code.vuplus.com Git - vuplus_xbmc/blob - xbmc/music/infoscanner/MusicInfoScanner.cpp

   1 /*
   2  *      Copyright (C) 2005-2013 Team XBMC
   3  *      http://www.xbmc.org
   4  *
   5  *  This Program is free software; you can redistribute it and/or modify
   6  *  it under the terms of the GNU General Public License as published by
   7  *  the Free Software Foundation; either version 2, or (at your option)
   8  *  any later version.
   9  *
  10  *  This Program is distributed in the hope that it will be useful,
  11  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  *  GNU General Public License for more details.
  14  *
  15  *  You should have received a copy of the GNU General Public License
  16  *  along with XBMC; see the file COPYING.  If not, see
  17  *  <http://www.gnu.org/licenses/>.
  18  *
  19  */
  20
  21 #include "threads/SystemClock.h"
  22 #include "MusicInfoScanner.h"
  23 #include "music/tags/MusicInfoTagLoaderFactory.h"
  24 #include "MusicAlbumInfo.h"
  25 #include "MusicInfoScraper.h"
  26 #include "filesystem/MusicDatabaseDirectory.h"
  27 #include "filesystem/MusicDatabaseDirectory/DirectoryNode.h"
  28 #include "Util.h"
  29 #include "utils/md5.h"
  30 #include "GUIInfoManager.h"
  31 #include "utils/Variant.h"
  32 #include "NfoFile.h"
  33 #include "music/tags/MusicInfoTag.h"
  34 #include "guilib/GUIWindowManager.h"
  35 #include "dialogs/GUIDialogExtendedProgressBar.h"
  36 #include "dialogs/GUIDialogProgress.h"
  37 #include "dialogs/GUIDialogSelect.h"
  38 #include "guilib/GUIKeyboardFactory.h"
  39 #include "filesystem/File.h"
  40 #include "filesystem/Directory.h"
  41 #include "settings/AdvancedSettings.h"
  42 #include "settings/Settings.h"
  43 #include "FileItem.h"
  44 #include "guilib/LocalizeStrings.h"
  45 #include "utils/StringUtils.h"
  46 #include "utils/TimeUtils.h"
  47 #include "utils/log.h"
  48 #include "utils/URIUtils.h"
  49 #include "TextureCache.h"
  50 #include "music/MusicThumbLoader.h"
  51 #include "interfaces/AnnouncementManager.h"
  52 #include "GUIUserMessages.h"
  53 #include "addons/AddonManager.h"
  54 #include "addons/Scraper.h"
  55
  56 #include <algorithm>
  57
  58 using namespace std;
  59 using namespace MUSIC_INFO;
  60 using namespace XFILE;
  61 using namespace MUSICDATABASEDIRECTORY;
  62 using namespace MUSIC_GRABBER;
  63 using namespace ADDON;
  64
  65 CMusicInfoScanner::CMusicInfoScanner() : CThread("MusicInfoScanner"), m_fileCountReader(this, "MusicFileCounter")
  66 {
  67   m_bRunning = false;
  68   m_showDialog = false;
  69   m_handle = NULL;
  70   m_bCanInterrupt = false;
  71   m_currentItem=0;
  72   m_itemCount=0;
  73   m_flags = 0;
  74 }
  75
  76 CMusicInfoScanner::~CMusicInfoScanner()
  77 {
  78 }
  79
  80 void CMusicInfoScanner::Process()
  81 {
  82   ANNOUNCEMENT::CAnnouncementManager::Announce(ANNOUNCEMENT::AudioLibrary, "xbmc", "OnScanStarted");
  83   try
  84   {
  85     unsigned int tick = XbmcThreads::SystemClockMillis();
  86
  87     m_musicDatabase.Open();
  88
  89     if (m_showDialog && !CSettings::Get().GetBool("musiclibrary.backgroundupdate"))
  90     {
  91       CGUIDialogExtendedProgressBar* dialog =
  92         (CGUIDialogExtendedProgressBar*)g_windowManager.GetWindow(WINDOW_DIALOG_EXT_PROGRESS);
  93       m_handle = dialog->GetHandle(g_localizeStrings.Get(314));
  94     }
  95
  96     m_bCanInterrupt = true;
  97
  98     if (m_scanType == 0) // load info from files
  99     {
 100       CLog::Log(LOGDEBUG, "%s - Starting scan", __FUNCTION__);
 101
 102       if (m_handle)
 103         m_handle->SetTitle(g_localizeStrings.Get(505));
 104
 105       // Reset progress vars
 106       m_currentItem=0;
 107       m_itemCount=-1;
 108
 109       // Create the thread to count all files to be scanned
 110       SetPriority( GetMinPriority() );
 111       if (m_handle)
 112         m_fileCountReader.Create();
 113
 114       // Database operations should not be canceled
 115       // using Interupt() while scanning as it could
 116       // result in unexpected behaviour.
 117       m_bCanInterrupt = false;
 118       m_needsCleanup = false;
 119
 120       bool commit = false;
 121       bool cancelled = false;
 122       for (std::set<std::string>::const_iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end(); it++)
 123       {
 124         /*
 125          * A copy of the directory path is used because the path supplied is
 126          * immediately removed from the m_pathsToScan set in DoScan(). If the
 127          * reference points to the entry in the set a null reference error
 128          * occurs.
 129          */
 130         if (!DoScan(*it))
 131           cancelled = true;
 132         commit = !cancelled;
 133       }
 134
 135       if (commit)
 136       {
 137         g_infoManager.ResetLibraryBools();
 138
 139         if (m_needsCleanup)
 140         {
 141           if (m_handle)
 142           {
 143             m_handle->SetTitle(g_localizeStrings.Get(700));
 144             m_handle->SetText("");
 145           }
 146
 147           m_musicDatabase.CleanupOrphanedItems();
 148
 149           if (m_handle)
 150             m_handle->SetTitle(g_localizeStrings.Get(331));
 151
 152           m_musicDatabase.Compress(false);
 153         }
 154       }
 155
 156       m_fileCountReader.StopThread();
 157
 158       m_musicDatabase.EmptyCache();
 159
 160       tick = XbmcThreads::SystemClockMillis() - tick;
 161       CLog::Log(LOGNOTICE, "My Music: Scanning for music info using worker thread, operation took %s", StringUtils::SecondsToTimeString(tick / 1000).c_str());
 162     }
 163     if (m_scanType == 1) // load album info
 164     {
 165       for (std::set<std::string>::const_iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end(); ++it)
 166       {
 167         CQueryParams params;
 168         CDirectoryNode::GetDatabaseInfo(*it, params);
 169         if (m_musicDatabase.HasAlbumInfo(params.GetArtistId())) // should this be here?
 170           continue;
 171
 172         CAlbum album;
 173         m_musicDatabase.GetAlbumInfo(params.GetAlbumId(), album, &album.songs);
 174         if (m_handle)
 175         {
 176           float percentage = (float) std::distance(it, m_pathsToScan.end()) / m_pathsToScan.size();
 177           m_handle->SetText(StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator) + " - " + album.strAlbum);
 178           m_handle->SetPercentage(percentage);
 179         }
 180
 181         CMusicAlbumInfo albumInfo;
 182         UpdateDatabaseAlbumInfo(*it, albumInfo, false);
 183
 184         if (m_bStop)
 185           break;
 186       }
 187     }
 188     if (m_scanType == 2) // load artist info
 189     {
 190       for (std::set<std::string>::const_iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end(); ++it)
 191       {
 192         CQueryParams params;
 193         CDirectoryNode::GetDatabaseInfo(*it, params);
 194         if (m_musicDatabase.HasArtistInfo(params.GetArtistId())) // should this be here?
 195             continue;
 196
 197         CArtist artist;
 198         m_musicDatabase.GetArtistInfo(params.GetArtistId(), artist);
 199         m_musicDatabase.GetArtistPath(params.GetArtistId(), artist.strPath);
 200
 201         if (m_handle)
 202         {
 203           float percentage = (float) (std::distance(m_pathsToScan.begin(), it) / m_pathsToScan.size()) * 100;
 204           m_handle->SetText(artist.strArtist);
 205           m_handle->SetPercentage(percentage);
 206         }
 207
 208         CMusicArtistInfo artistInfo;
 209         UpdateDatabaseArtistInfo(*it, artistInfo, false);
 210
 211         if (m_bStop)
 212           break;
 213       }
 214     }
 215
 216   }
 217   catch (...)
 218   {
 219     CLog::Log(LOGERROR, "MusicInfoScanner: Exception while scanning.");
 220   }
 221   m_musicDatabase.Close();
 222   CLog::Log(LOGDEBUG, "%s - Finished scan", __FUNCTION__);
 223
 224   m_bRunning = false;
 225   ANNOUNCEMENT::CAnnouncementManager::Announce(ANNOUNCEMENT::AudioLibrary, "xbmc", "OnScanFinished");
 226
 227   // we need to clear the musicdb cache and update any active lists
 228   CUtil::DeleteMusicDatabaseDirectoryCache();
 229   CGUIMessage msg(GUI_MSG_SCAN_FINISHED, 0, 0, 0);
 230   g_windowManager.SendThreadMessage(msg);
 231
 232   if (m_handle)
 233     m_handle->MarkFinished();
 234   m_handle = NULL;
 235 }
 236
 237 void CMusicInfoScanner::Start(const CStdString& strDirectory, int flags)
 238 {
 239   m_fileCountReader.StopThread();
 240   StopThread();
 241   m_pathsToScan.clear();
 242   m_flags = flags;
 243
 244   if (strDirectory.IsEmpty())
 245   { // scan all paths in the database.  We do this by scanning all paths in the db, and crossing them off the list as
 246     // we go.
 247     m_musicDatabase.Open();
 248     m_musicDatabase.GetPaths(m_pathsToScan);
 249     m_musicDatabase.Close();
 250   }
 251   else
 252     m_pathsToScan.insert(strDirectory);
 253
 254   m_scanType = 0;
 255   Create();
 256   m_bRunning = true;
 257 }
 258
 259 void CMusicInfoScanner::FetchAlbumInfo(const CStdString& strDirectory,
 260                                        bool refresh)
 261 {
 262   m_fileCountReader.StopThread();
 263   StopThread();
 264   m_pathsToScan.clear();
 265
 266   CFileItemList items;
 267   if (strDirectory.IsEmpty())
 268   {
 269     m_musicDatabase.Open();
 270     m_musicDatabase.GetAlbumsNav("musicdb://albums/", items);
 271     m_musicDatabase.Close();
 272   }
 273   else
 274   {
 275     if (URIUtils::HasSlashAtEnd(strDirectory)) // directory
 276       CDirectory::GetDirectory(strDirectory,items);
 277     else
 278     {
 279       CFileItemPtr item(new CFileItem(strDirectory,false));
 280       items.Add(item);
 281     }
 282   }
 283
 284   m_musicDatabase.Open();
 285   for (int i=0;i<items.Size();++i)
 286   {
 287     if (CMusicDatabaseDirectory::IsAllItem(items[i]->GetPath()) || items[i]->IsParentFolder())
 288       continue;
 289
 290     m_pathsToScan.insert(items[i]->GetPath());
 291     if (refresh)
 292     {
 293       m_musicDatabase.DeleteAlbumInfo(items[i]->GetMusicInfoTag()->GetDatabaseId());
 294     }
 295   }
 296   m_musicDatabase.Close();
 297
 298   m_scanType = 1;
 299   Create();
 300   m_bRunning = true;
 301 }
 302
 303 void CMusicInfoScanner::FetchArtistInfo(const CStdString& strDirectory,
 304                                         bool refresh)
 305 {
 306   m_fileCountReader.StopThread();
 307   StopThread();
 308   m_pathsToScan.clear();
 309   CFileItemList items;
 310
 311   if (strDirectory.IsEmpty())
 312   {
 313     m_musicDatabase.Open();
 314     m_musicDatabase.GetArtistsNav("musicdb://artists/", items, false, -1);
 315     m_musicDatabase.Close();
 316   }
 317   else
 318   {
 319     if (URIUtils::HasSlashAtEnd(strDirectory)) // directory
 320       CDirectory::GetDirectory(strDirectory,items);
 321     else
 322     {
 323       CFileItemPtr newItem(new CFileItem(strDirectory,false));
 324       items.Add(newItem);
 325     }
 326   }
 327
 328   m_musicDatabase.Open();
 329   for (int i=0;i<items.Size();++i)
 330   {
 331     if (CMusicDatabaseDirectory::IsAllItem(items[i]->GetPath()) || items[i]->IsParentFolder())
 332       continue;
 333
 334     m_pathsToScan.insert(items[i]->GetPath());
 335     if (refresh)
 336     {
 337       m_musicDatabase.DeleteArtistInfo(items[i]->GetMusicInfoTag()->GetDatabaseId());
 338     }
 339   }
 340   m_musicDatabase.Close();
 341
 342   m_scanType = 2;
 343   Create();
 344   m_bRunning = true;
 345 }
 346
 347 bool CMusicInfoScanner::IsScanning()
 348 {
 349   return m_bRunning;
 350 }
 351
 352 void CMusicInfoScanner::Stop()
 353 {
 354   if (m_bCanInterrupt)
 355     m_musicDatabase.Interupt();
 356
 357   StopThread(false);
 358 }
 359
 360 static void OnDirectoryScanned(const CStdString& strDirectory)
 361 {
 362   CGUIMessage msg(GUI_MSG_DIRECTORY_SCANNED, 0, 0, 0);
 363   msg.SetStringParam(strDirectory);
 364   g_windowManager.SendThreadMessage(msg);
 365 }
 366
 367 static CStdString Prettify(const CStdString& strDirectory)
 368 {
 369   CURL url(strDirectory);
 370   CStdString strStrippedPath = url.GetWithoutUserDetails();
 371   CURL::Decode(strStrippedPath);
 372
 373   return strStrippedPath;
 374 }
 375
 376 bool CMusicInfoScanner::DoScan(const CStdString& strDirectory)
 377 {
 378   if (m_handle)
 379     m_handle->SetText(Prettify(strDirectory));
 380
 381   // Discard all excluded files defined by m_musicExcludeRegExps
 382   CStdStringArray regexps = g_advancedSettings.m_audioExcludeFromScanRegExps;
 383   if (CUtil::ExcludeFileOrFolder(strDirectory, regexps))
 384     return true;
 385
 386   // load subfolder
 387   CFileItemList items;
 388   CDirectory::GetDirectory(strDirectory, items, g_advancedSettings.m_musicExtensions + "|.jpg|.tbn|.lrc|.cdg");
 389
 390   // sort and get the path hash.  Note that we don't filter .cue sheet items here as we want
 391   // to detect changes in the .cue sheet as well.  The .cue sheet items only need filtering
 392   // if we have a changed hash.
 393   items.Sort(SORT_METHOD_LABEL, SortOrderAscending);
 394   CStdString hash;
 395   GetPathHash(items, hash);
 396
 397   // check whether we need to rescan or not
 398   CStdString dbHash;
 399   if ((m_flags & SCAN_RESCAN) || !m_musicDatabase.GetPathHash(strDirectory, dbHash) || dbHash != hash)
 400   { // path has changed - rescan
 401     if (dbHash.IsEmpty())
 402       CLog::Log(LOGDEBUG, "%s Scanning dir '%s' as not in the database", __FUNCTION__, strDirectory.c_str());
 403     else
 404       CLog::Log(LOGDEBUG, "%s Rescanning dir '%s' due to change", __FUNCTION__, strDirectory.c_str());
 405
 406     // filter items in the sub dir (for .cue sheet support)
 407     items.FilterCueItems();
 408     items.Sort(SORT_METHOD_LABEL, SortOrderAscending);
 409
 410     // and then scan in the new information
 411     if (RetrieveMusicInfo(strDirectory, items) > 0)
 412     {
 413       if (m_handle)
 414         OnDirectoryScanned(strDirectory);
 415     }
 416
 417     // save information about this folder
 418     m_musicDatabase.SetPathHash(strDirectory, hash);
 419   }
 420   else
 421   { // path is the same - no need to rescan
 422     CLog::Log(LOGDEBUG, "%s Skipping dir '%s' due to no change", __FUNCTION__, strDirectory.c_str());
 423     m_currentItem += CountFiles(items, false);  // false for non-recursive
 424
 425     // updated the dialog with our progress
 426     if (m_handle)
 427     {
 428       if (m_itemCount>0)
 429         m_handle->SetPercentage(m_currentItem/(float)m_itemCount*100);
 430       OnDirectoryScanned(strDirectory);
 431     }
 432   }
 433
 434   // now scan the subfolders
 435   for (int i = 0; i < items.Size(); ++i)
 436   {
 437     CFileItemPtr pItem = items[i];
 438
 439     if (m_bStop)
 440       break;
 441     // if we have a directory item (non-playlist) we then recurse into that folder
 442     if (pItem->m_bIsFolder && !pItem->IsParentFolder() && !pItem->IsPlayList())
 443     {
 444       CStdString strPath=pItem->GetPath();
 445       if (!DoScan(strPath))
 446       {
 447         m_bStop = true;
 448       }
 449     }
 450   }
 451
 452   return !m_bStop;
 453 }
 454
 455 INFO_RET CMusicInfoScanner::ScanTags(const CFileItemList& items, CFileItemList& scannedItems)
 456 {
 457   CStdStringArray regexps = g_advancedSettings.m_audioExcludeFromScanRegExps;
 458
 459   for (int i = 0; i < items.Size(); ++i)
 460   {
 461     if (m_bStop)
 462       return INFO_CANCELLED;
 463
 464     CFileItemPtr pItem = items[i];
 465
 466     if (CUtil::ExcludeFileOrFolder(pItem->GetPath(), regexps))
 467       continue;
 468
 469     if (pItem->m_bIsFolder || pItem->IsPlayList() || pItem->IsPicture() || pItem->IsLyrics())
 470       continue;
 471
 472     m_currentItem++;
 473
 474     CMusicInfoTag& tag = *pItem->GetMusicInfoTag();
 475     if (!tag.Loaded())
 476     {
 477       auto_ptr<IMusicInfoTagLoader> pLoader (CMusicInfoTagLoaderFactory::CreateLoader(pItem->GetPath()));
 478       if (NULL != pLoader.get())
 479         pLoader->Load(pItem->GetPath(), tag);
 480     }
 481
 482     if (m_handle && m_itemCount>0)
 483       m_handle->SetPercentage(m_currentItem/(float)m_itemCount*100);
 484
 485     if (!tag.Loaded())
 486     {
 487       CLog::Log(LOGDEBUG, "%s - No tag found for: %s", __FUNCTION__, pItem->GetPath().c_str());
 488       continue;
 489     }
 490     scannedItems.Add(pItem);
 491   }
 492   return INFO_ADDED;
 493 }
 494
 495 void CMusicInfoScanner::FileItemsToAlbums(CFileItemList& items, VECALBUMS& albums, MAPSONGS* songsMap /* = NULL */)
 496 {
 497   for (int i = 0; i < items.Size(); ++i)
 498   {
 499     CFileItemPtr pItem = items[i];
 500     CMusicInfoTag& tag = *pItem->GetMusicInfoTag();
 501     CSong song(*pItem);
 502
 503     // keep the db-only fields intact on rescan...
 504     if (songsMap != NULL)
 505     {
 506       MAPSONGS::iterator it = songsMap->find(pItem->GetPath());
 507       if (it != songsMap->end())
 508       {
 509         song.iTimesPlayed = it->second.iTimesPlayed;
 510         song.lastPlayed = it->second.lastPlayed;
 511         song.iKaraokeNumber = it->second.iKaraokeNumber;
 512         if (song.rating == '0')    song.rating = it->second.rating;
 513         if (song.strThumb.empty()) song.strThumb = it->second.strThumb;
 514       }
 515     }
 516
 517     if (!tag.GetMusicBrainzArtistID().empty())
 518     {
 519       for (vector<string>::const_iterator it = tag.GetMusicBrainzArtistID().begin(); it != tag.GetMusicBrainzArtistID().end(); ++it)
 520       {
 521         CStdString strJoinPhrase = (it == --tag.GetMusicBrainzArtistID().end() ? "" : g_advancedSettings.m_musicItemSeparator);
 522         CArtistCredit mbartist(tag.GetArtist().empty() ? "" : tag.GetArtist()[0], *it, strJoinPhrase);
 523         song.artistCredits.push_back(mbartist);
 524       }
 525       song.artist = tag.GetArtist();
 526     }
 527     else
 528     {
 529       for (vector<string>::const_iterator it = tag.GetArtist().begin(); it != tag.GetArtist().end(); ++it)
 530       {
 531         CStdString strJoinPhrase = (it == --tag.GetArtist().end() ? "" : g_advancedSettings.m_musicItemSeparator);
 532         CArtistCredit nonmbartist(*it, strJoinPhrase);
 533         song.artistCredits.push_back(nonmbartist);
 534       }
 535       song.artist = tag.GetArtist();
 536     }
 537
 538     bool found = false;
 539     for (VECALBUMS::iterator it = albums.begin(); it != albums.end(); ++it)
 540     {
 541       if (it->strAlbum == tag.GetAlbum() && it->strMusicBrainzAlbumID == tag.GetMusicBrainzAlbumID())
 542       {
 543         it->songs.push_back(song);
 544         found = true;
 545       }
 546     }
 547     if (!found)
 548     {
 549       CAlbum album(*pItem.get());
 550       if (!tag.GetMusicBrainzAlbumArtistID().empty())
 551       {
 552         for (vector<string>::const_iterator it = tag.GetMusicBrainzAlbumArtistID().begin(); it != tag.GetMusicBrainzAlbumArtistID().end(); ++it)
 553         {
 554           // Picard always stored the display artist string in the first artist slot, no need to split it
 555           CStdString strJoinPhrase = (it == --tag.GetMusicBrainzAlbumArtistID().end() ? "" : g_advancedSettings.m_musicItemSeparator);
 556           CArtistCredit mbartist(tag.GetAlbumArtist().empty() ? "" : tag.GetAlbumArtist()[0], *it, strJoinPhrase);
 557           album.artistCredits.push_back(mbartist);
 558         }
 559         album.artist = tag.GetAlbumArtist();
 560       }
 561       else
 562       {
 563         for (vector<string>::const_iterator it = tag.GetAlbumArtist().begin(); it != tag.GetAlbumArtist().end(); ++it)
 564         {
 565           CStdString strJoinPhrase = (it == --tag.GetAlbumArtist().end() ? "" : g_advancedSettings.m_musicItemSeparator);
 566           CArtistCredit nonmbartist(*it, strJoinPhrase);
 567           album.artistCredits.push_back(nonmbartist);
 568         }
 569         album.artist = tag.GetAlbumArtist();
 570       }
 571       album.songs.push_back(song);
 572       albums.push_back(album);
 573     }
 574   }
 575 }
 576
 577 int CMusicInfoScanner::RetrieveMusicInfo(const CStdString& strDirectory, CFileItemList& items)
 578 {
 579   MAPSONGS songsMap;
 580
 581   // get all information for all files in current directory from database, and remove them
 582   if (m_musicDatabase.RemoveSongsFromPath(strDirectory, songsMap))
 583     m_needsCleanup = true;
 584
 585   CFileItemList scannedItems;
 586   if (ScanTags(items, scannedItems) == INFO_CANCELLED)
 587     return 0;
 588
 589   VECALBUMS albums;
 590   FileItemsToAlbums(scannedItems, albums, &songsMap);
 591   if (!(m_flags & SCAN_ONLINE))
 592     FixupAlbums(albums);
 593   FindArtForAlbums(albums, items.GetPath());
 594
 595   int numAdded = 0;
 596   ADDON::AddonPtr addon;
 597   ADDON::ScraperPtr albumScraper;
 598   ADDON::ScraperPtr artistScraper;
 599   if(ADDON::CAddonMgr::Get().GetDefault(ADDON::ADDON_SCRAPER_ALBUMS, addon))
 600     albumScraper = boost::dynamic_pointer_cast<ADDON::CScraper>(addon);
 601
 602   if(ADDON::CAddonMgr::Get().GetDefault(ADDON::ADDON_SCRAPER_ARTISTS, addon))
 603     artistScraper = boost::dynamic_pointer_cast<ADDON::CScraper>(addon);
 604
 605   // Add each album
 606   for (VECALBUMS::iterator album = albums.begin(); album != albums.end(); ++album)
 607   {
 608     if (m_bStop)
 609       break;
 610
 611     album->strPath = strDirectory;
 612     m_musicDatabase.BeginTransaction();
 613
 614     // Check if the album has already been downloaded or failed
 615     map<CAlbum, CAlbum>::iterator cachedAlbum = m_albumCache.find(*album);
 616     if (cachedAlbum == m_albumCache.end())
 617     {
 618       // No - download the information
 619       CMusicAlbumInfo albumInfo;
 620       INFO_RET albumDownloadStatus = INFO_NOT_FOUND;
 621       if ((m_flags & SCAN_ONLINE) && albumScraper)
 622         albumDownloadStatus = DownloadAlbumInfo(*album, albumScraper, albumInfo);
 623
 624       if (albumDownloadStatus == INFO_ADDED || albumDownloadStatus == INFO_HAVE_ALREADY)
 625       {
 626         CAlbum &downloadedAlbum = albumInfo.GetAlbum();
 627         downloadedAlbum.idAlbum = m_musicDatabase.AddAlbum(downloadedAlbum.strAlbum,
 628                                                            downloadedAlbum.strMusicBrainzAlbumID,
 629                                                            downloadedAlbum.GetArtistString(),
 630                                                            downloadedAlbum.GetGenreString(),
 631                                                            downloadedAlbum.iYear,
 632                                                            downloadedAlbum.bCompilation);
 633         m_musicDatabase.SetAlbumInfo(downloadedAlbum.idAlbum,
 634                                      downloadedAlbum,
 635                                      downloadedAlbum.songs);
 636         m_musicDatabase.SetArtForItem(downloadedAlbum.idAlbum,
 637                                       "album", album->art);
 638         GetAlbumArtwork(downloadedAlbum.idAlbum, downloadedAlbum);
 639         m_albumCache.insert(make_pair(*album, albumInfo.GetAlbum()));
 640       }
 641       else if (albumDownloadStatus == INFO_CANCELLED)
 642         break;
 643       else
 644       {
 645         // No download info, fallback to already gathered (eg. local) information/art (if any)
 646         album->idAlbum = m_musicDatabase.AddAlbum(album->strAlbum,
 647                                                   album->strMusicBrainzAlbumID,
 648                                                   album->GetArtistString(),
 649                                                   album->GetGenreString(),
 650                                                   album->iYear,
 651                                                   album->bCompilation);
 652         if (!album->art.empty())
 653           m_musicDatabase.SetArtForItem(album->idAlbum,
 654                                         "album", album->art);
 655         m_albumCache.insert(make_pair(*album, *album));
 656       }
 657
 658       // Update the cache pointer with our newly created info
 659       cachedAlbum = m_albumCache.find(*album);
 660     }
 661
 662     if (m_bStop)
 663       break;
 664
 665     // Add the album artists
 666     for (VECARTISTCREDITS::iterator artistCredit = cachedAlbum->second.artistCredits.begin(); artistCredit != cachedAlbum->second.artistCredits.end(); ++artistCredit)
 667     {
 668       if (m_bStop)
 669         break;
 670
 671       // Check if the artist has already been downloaded or failed
 672       map<CArtistCredit, CArtist>::iterator cachedArtist = m_artistCache.find(*artistCredit);
 673       if (cachedArtist == m_artistCache.end())
 674       {
 675         CArtist artistTmp;
 676         artistTmp.strArtist = artistCredit->GetArtist();
 677         artistTmp.strMusicBrainzArtistID = artistCredit->GetMusicBrainzArtistID();
 678         URIUtils::GetParentPath(album->strPath, artistTmp.strPath);
 679
 680         // No - download the information
 681         CMusicArtistInfo artistInfo;
 682         INFO_RET artistDownloadStatus = INFO_NOT_FOUND;
 683         if ((m_flags & SCAN_ONLINE) && artistScraper)
 684           artistDownloadStatus = DownloadArtistInfo(artistTmp, artistScraper, artistInfo);
 685
 686         if (artistDownloadStatus == INFO_ADDED || artistDownloadStatus == INFO_HAVE_ALREADY)
 687         {
 688           CArtist &downloadedArtist = artistInfo.GetArtist();
 689           downloadedArtist.idArtist = m_musicDatabase.AddArtist(downloadedArtist.strArtist,
 690                                                                 downloadedArtist.strMusicBrainzArtistID);
 691           m_musicDatabase.SetArtistInfo(downloadedArtist.idArtist,
 692                                         downloadedArtist);
 693
 694           URIUtils::GetParentPath(album->strPath, downloadedArtist.strPath);
 695           map<string, string> artwork = GetArtistArtwork(downloadedArtist);
 696           // check thumb stuff
 697           m_musicDatabase.SetArtForItem(downloadedArtist.idArtist, "artist", artwork);
 698           m_artistCache.insert(make_pair(*artistCredit, downloadedArtist));
 699         }
 700         else if (artistDownloadStatus == INFO_CANCELLED)
 701           break;
 702         else
 703         {
 704           // Cache the lookup failure so we don't retry
 705           artistTmp.idArtist = m_musicDatabase.AddArtist(artistCredit->GetArtist(), artistCredit->GetMusicBrainzArtistID());
 706           m_artistCache.insert(make_pair(*artistCredit, artistTmp));
 707         }
 708
 709         // Update the cache pointer with our newly created info
 710         cachedArtist = m_artistCache.find(*artistCredit);
 711       }
 712
 713       m_musicDatabase.AddAlbumArtist(cachedArtist->second.idArtist,
 714                                      cachedAlbum->second.idAlbum,
 715                                      artistCredit->GetJoinPhrase(),
 716                                      artistCredit == album->artistCredits.begin() ? false : true,
 717                                      std::distance(cachedAlbum->second.artistCredits.begin(), artistCredit));
 718     }
 719
 720     if (m_bStop)
 721       break;
 722
 723     for (VECSONGS::iterator song = album->songs.begin(); song != album->songs.end(); ++song)
 724     {
 725       song->idAlbum = cachedAlbum->second.idAlbum;
 726       song->idSong = m_musicDatabase.AddSong(cachedAlbum->second.idAlbum,
 727                                              song->strTitle, song->strMusicBrainzTrackID,
 728                                              song->strFileName, song->strComment,
 729                                              song->strThumb,
 730                                              song->artist, song->genre,
 731                                              song->iTrack, song->iDuration, song->iYear,
 732                                              song->iTimesPlayed, song->iStartOffset,
 733                                              song->iEndOffset,
 734                                              song->lastPlayed,
 735                                              song->rating,
 736                                              song->iKaraokeNumber);
 737       for (VECARTISTCREDITS::iterator artistCredit = song->artistCredits.begin(); artistCredit != song->artistCredits.end(); ++artistCredit)
 738       {
 739         if (m_bStop)
 740           break;
 741
 742         // Check if the artist has already been downloaded or failed
 743         map<CArtistCredit, CArtist>::iterator cachedArtist = m_artistCache.find(*artistCredit);
 744         if (cachedArtist == m_artistCache.end())
 745         {
 746           CArtist artistTmp;
 747           artistTmp.strArtist = artistCredit->GetArtist();
 748           artistTmp.strMusicBrainzArtistID = artistCredit->GetMusicBrainzArtistID();
 749           URIUtils::GetParentPath(album->strPath, artistTmp.strPath); // FIXME
 750
 751           // No - download the information
 752           CMusicArtistInfo artistInfo;
 753           INFO_RET artistDownloadStatus = INFO_NOT_FOUND;
 754           if ((m_flags & SCAN_ONLINE) && artistScraper)
 755             artistDownloadStatus = DownloadArtistInfo(artistTmp, artistScraper, artistInfo);
 756
 757           if (artistDownloadStatus == INFO_ADDED || artistDownloadStatus == INFO_HAVE_ALREADY)
 758           {
 759             CArtist &downloadedArtist = artistInfo.GetArtist();
 760             downloadedArtist.idArtist = m_musicDatabase.AddArtist(downloadedArtist.strArtist,
 761                                                                   downloadedArtist.strMusicBrainzArtistID);
 762             m_musicDatabase.SetArtistInfo(downloadedArtist.idArtist,
 763                                           downloadedArtist);
 764             // check thumb stuff
 765             URIUtils::GetParentPath(album->strPath, downloadedArtist.strPath);
 766             map<string, string> artwork = GetArtistArtwork(downloadedArtist);
 767             m_musicDatabase.SetArtForItem(downloadedArtist.idArtist, "artist", artwork);
 768             m_artistCache.insert(make_pair(*artistCredit, downloadedArtist));
 769           }
 770           else if (artistDownloadStatus == INFO_CANCELLED)
 771             break;
 772           else
 773           {
 774             // Cache the lookup failure so we don't retry
 775             artistTmp.idArtist = m_musicDatabase.AddArtist(artistCredit->GetArtist(), artistCredit->GetMusicBrainzArtistID());
 776             m_artistCache.insert(make_pair(*artistCredit, artistTmp));
 777           }
 778
 779           // Update the cache pointer with our newly created info
 780           cachedArtist = m_artistCache.find(*artistCredit);
 781         }
 782
 783         m_musicDatabase.AddSongArtist(cachedArtist->second.idArtist,
 784                                       song->idSong,
 785                                       g_advancedSettings.m_musicItemSeparator, // we don't have song artist breakdowns from scrapers, yet
 786                                       artistCredit == song->artistCredits.begin() ? false : true,
 787                                       std::distance(song->artistCredits.begin(), artistCredit));
 788       }
 789     }
 790
 791     if (m_bStop)
 792       break;
 793
 794     // Commit the album to the DB
 795     m_musicDatabase.CommitTransaction();
 796     numAdded += album->songs.size();
 797   }
 798
 799   if (m_bStop)
 800     m_musicDatabase.RollbackTransaction();
 801
 802   if (m_handle)
 803     m_handle->SetTitle(g_localizeStrings.Get(505));
 804
 805   return numAdded;
 806 }
 807
 808 static bool SortSongsByTrack(const CSong& song, const CSong& song2)
 809 {
 810   return song.iTrack < song2.iTrack;
 811 }
 812
 813 void CMusicInfoScanner::FixupAlbums(VECALBUMS &albums)
 814 {
 815   /*
 816    Step 2: Split into unique albums based on album name and album artist
 817    In the case where the album artist is unknown, we use the primary artist
 818    (i.e. first artist from each song).
 819    */
 820   for (VECALBUMS::iterator album = albums.begin(); album != albums.end(); ++album)
 821   {
 822     /*
 823      * If we have a valid MusicBrainz tag for the album, assume everything
 824      * is okay with our tags, as Picard should set everything up correctly.
 825      */
 826     if (!album->strMusicBrainzAlbumID.IsEmpty())
 827       continue;
 828
 829     VECSONGS &songs = album->songs;
 830     // sort the songs by tracknumber to identify duplicate track numbers
 831     sort(songs.begin(), songs.end(), SortSongsByTrack);
 832
 833     // map the songs to their primary artists
 834     bool tracksOverlap = false;
 835     bool hasAlbumArtist = false;
 836     bool isCompilation = true;
 837
 838     map<string, vector<CSong *> > artists;
 839     for (VECSONGS::iterator song = songs.begin(); song != songs.end(); ++song)
 840     {
 841       // test for song overlap
 842       if (song != songs.begin() && song->iTrack == (song - 1)->iTrack)
 843         tracksOverlap = true;
 844
 845       if (!song->bCompilation)
 846         isCompilation = false;
 847
 848       // get primary artist
 849       string primary;
 850       if (!song->albumArtist.empty())
 851       {
 852         primary = song->albumArtist[0];
 853         hasAlbumArtist = true;
 854       }
 855       else if (!song->artist.empty())
 856         primary = song->artist[0];
 857
 858       // add to the artist map
 859       artists[primary].push_back(&(*song));
 860     }
 861
 862     /*
 863      We have a compilation if
 864      1. album name is non-empty AND
 865      2a. no tracks overlap OR
 866      2b. all tracks are marked as part of compilation AND
 867      3a. a unique primary artist is specified as "various" or "various artists" OR
 868      3b. we have at least two primary artists and no album artist specified.
 869      */
 870     bool compilation = !album->strAlbum.empty() && (isCompilation || !tracksOverlap); // 1+2b+2a
 871     if (artists.size() == 1)
 872     {
 873       string artist = artists.begin()->first; StringUtils::ToLower(artist);
 874       if (!StringUtils::EqualsNoCase(artist, "various") &&
 875           !StringUtils::EqualsNoCase(artist, "various artists")) // 3a
 876         compilation = false;
 877     }
 878     else if (hasAlbumArtist) // 3b
 879       compilation = false;
 880
 881     if (compilation)
 882     {
 883       CLog::Log(LOGDEBUG, "Album '%s' is a compilation as there's no overlapping tracks and %s", album->strAlbum.c_str(), hasAlbumArtist ? "the album artist is 'Various'" : "there is more than one unique artist");
 884       artists.clear();
 885       std::string various = g_localizeStrings.Get(340); // Various Artists
 886       vector<string> va; va.push_back(various);
 887       for (VECSONGS::iterator song = songs.begin(); song != songs.end(); ++song)
 888       {
 889         song->albumArtist = va;
 890         artists[various].push_back(&(*song));
 891       }
 892     }
 893
 894     /*
 895      Step 3: Find the common albumartist for each song and assign
 896      albumartist to those tracks that don't have it set.
 897      */
 898     for (map<string, vector<CSong *> >::iterator j = artists.begin(); j != artists.end(); ++j)
 899     {
 900       // find the common artist for these songs
 901       vector<CSong *> &artistSongs = j->second;
 902       vector<string> common = artistSongs.front()->albumArtist.empty() ? artistSongs.front()->artist : artistSongs.front()->albumArtist;
 903       for (vector<CSong *>::iterator k = artistSongs.begin() + 1; k != artistSongs.end(); ++k)
 904       {
 905         unsigned int match = 0;
 906         vector<string> &compare = (*k)->albumArtist.empty() ? (*k)->artist : (*k)->albumArtist;
 907         for (; match < common.size() && match < compare.size(); match++)
 908         {
 909           if (compare[match] != common[match])
 910             break;
 911         }
 912         common.erase(common.begin() + match, common.end());
 913       }
 914
 915       /*
 916        Step 4: Assign the album artist for each song that doesn't have it set
 917        and add to the album vector
 918        */
 919       album->artistCredits.clear();
 920       for (vector<string>::iterator it = common.begin(); it != common.end(); ++it)
 921       {
 922         CStdString strJoinPhrase = (it == --common.end() ? "" : g_advancedSettings.m_musicItemSeparator);
 923         CArtistCredit artistCredit(*it, strJoinPhrase);
 924         album->artistCredits.push_back(artistCredit);
 925       }
 926       album->bCompilation = compilation;
 927       for (vector<CSong *>::iterator k = artistSongs.begin(); k != artistSongs.end(); ++k)
 928       {
 929         if ((*k)->albumArtist.empty())
 930           (*k)->albumArtist = common;
 931         // TODO: in future we may wish to union up the genres, for now we assume they're the same
 932         if (album->genre.empty())
 933           album->genre = (*k)->genre;
 934         //       in addition, we may want to use year as discriminating for albums
 935         if (album->iYear == 0)
 936           album->iYear = (*k)->iYear;
 937       }
 938     }
 939   }
 940 }
 941
 942 void CMusicInfoScanner::FindArtForAlbums(VECALBUMS &albums, const CStdString &path)
 943 {
 944   /*
 945    If there's a single album in the folder, then art can be taken from
 946    the folder art.
 947    */
 948   std::string albumArt;
 949   if (albums.size() == 1)
 950   {
 951     CFileItem album(path, true);
 952     albumArt = album.GetUserMusicThumb(true);
 953     if (!albumArt.empty())
 954       albums[0].art["thumb"] = albumArt;
 955   }
 956   for (VECALBUMS::iterator i = albums.begin(); i != albums.end(); ++i)
 957   {
 958     CAlbum &album = *i;
 959
 960     if (albums.size() != 1)
 961       albumArt = "";
 962
 963     /*
 964      Find art that is common across these items
 965      If we find a single art image we treat it as the album art
 966      and discard song art else we use first as album art and
 967      keep everything as song art.
 968      */
 969     bool singleArt = true;
 970     CSong *art = NULL;
 971     for (VECSONGS::iterator k = album.songs.begin(); k != album.songs.end(); ++k)
 972     {
 973       CSong &song = *k;
 974       if (song.HasArt())
 975       {
 976         if (art && !art->ArtMatches(song))
 977         {
 978           singleArt = false;
 979           break;
 980         }
 981         if (!art)
 982           art = &song;
 983       }
 984     }
 985
 986     /*
 987       assign the first art found to the album - better than no art at all
 988     */
 989
 990     if (art && albumArt.empty())
 991     {
 992       if (!art->strThumb.empty())
 993         albumArt = art->strThumb;
 994       else
 995         albumArt = CTextureCache::GetWrappedImageURL(art->strFileName, "music");
 996     }
 997
 998     if (!albumArt.empty())
 999       album.art["thumb"] = albumArt;
1000
1001     if (singleArt)
1002     { //if singleArt then we can clear the artwork for all songs
1003       for (VECSONGS::iterator k = album.songs.begin(); k != album.songs.end(); ++k)
1004         k->strThumb.clear();
1005     }
1006     else
1007     { // more than one piece of art was found for these songs, so cache per song
1008       for (VECSONGS::iterator k = album.songs.begin(); k != album.songs.end(); ++k)
1009       {
1010         if (k->strThumb.empty() && !k->embeddedArt.empty())
1011           k->strThumb = CTextureCache::GetWrappedImageURL(k->strFileName, "music");
1012       }
1013     }
1014   }
1015   if (albums.size() == 1 && !albumArt.empty())
1016   {
1017     // assign to folder thumb as well
1018     CFileItem albumItem(path, true);
1019     CMusicThumbLoader loader;
1020     loader.SetCachedImage(albumItem, "thumb", albumArt);
1021   }
1022 }
1023
1024 int CMusicInfoScanner::GetPathHash(const CFileItemList &items, CStdString &hash)
1025 {
1026   // Create a hash based on the filenames, filesize and filedate.  Also count the number of files
1027   if (0 == items.Size()) return 0;
1028   XBMC::XBMC_MD5 md5state;
1029   int count = 0;
1030   for (int i = 0; i < items.Size(); ++i)
1031   {
1032     const CFileItemPtr pItem = items[i];
1033     md5state.append(pItem->GetPath());
1034     md5state.append((unsigned char *)&pItem->m_dwSize, sizeof(pItem->m_dwSize));
1035     FILETIME time = pItem->m_dateTime;
1036     md5state.append((unsigned char *)&time, sizeof(FILETIME));
1037     if (pItem->IsAudio() && !pItem->IsPlayList() && !pItem->IsNFO())
1038       count++;
1039   }
1040   md5state.getDigest(hash);
1041   return count;
1042 }
1043
1044 INFO_RET CMusicInfoScanner::UpdateDatabaseAlbumInfo(const CStdString& strPath, CMusicAlbumInfo& albumInfo, bool bAllowSelection, CGUIDialogProgress* pDialog /* = NULL */)
1045 {
1046   m_musicDatabase.Open();
1047   CQueryParams params;
1048   CDirectoryNode::GetDatabaseInfo(strPath, params);
1049
1050   if (params.GetAlbumId() == -1)
1051     return INFO_ERROR;
1052
1053   CAlbum album;
1054   m_musicDatabase.GetAlbumInfo(params.GetAlbumId(), album, &album.songs);
1055
1056   // find album info
1057   ADDON::ScraperPtr scraper;
1058   bool result = m_musicDatabase.GetScraperForPath(strPath, scraper, ADDON::ADDON_SCRAPER_ALBUMS);
1059
1060   m_musicDatabase.Close();
1061
1062   if (!result || !scraper)
1063     return INFO_ERROR;
1064
1065 loop:
1066   CLog::Log(LOGDEBUG, "%s downloading info for: %s", __FUNCTION__, album.strAlbum.c_str());
1067   INFO_RET albumDownloadStatus = DownloadAlbumInfo(album, scraper, albumInfo, pDialog);
1068   if (albumDownloadStatus == INFO_NOT_FOUND)
1069   {
1070     if (pDialog && bAllowSelection)
1071     {
1072       if (!CGUIKeyboardFactory::ShowAndGetInput(album.strAlbum, g_localizeStrings.Get(16011), false))
1073         return INFO_CANCELLED;
1074
1075       CStdString strTempArtist(StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
1076       if (!CGUIKeyboardFactory::ShowAndGetInput(strTempArtist, g_localizeStrings.Get(16025), false))
1077         return INFO_CANCELLED;
1078
1079       album.artist = StringUtils::Split(strTempArtist, g_advancedSettings.m_musicItemSeparator);
1080       goto loop;
1081     }
1082   }
1083   else if (albumDownloadStatus == INFO_ADDED)
1084   {
1085     m_musicDatabase.Open();
1086     m_musicDatabase.SetAlbumInfo(params.GetAlbumId(), albumInfo.GetAlbum(), albumInfo.GetAlbum().songs);
1087     GetAlbumArtwork(params.GetAlbumId(), albumInfo.GetAlbum());
1088     albumInfo.SetLoaded(true);
1089     m_musicDatabase.Close();
1090   }
1091   return albumDownloadStatus;
1092 }
1093
1094 INFO_RET CMusicInfoScanner::UpdateDatabaseArtistInfo(const CStdString& strPath, CMusicArtistInfo& artistInfo, bool bAllowSelection, CGUIDialogProgress* pDialog /* = NULL */)
1095 {
1096   m_musicDatabase.Open();
1097   CQueryParams params;
1098   CDirectoryNode::GetDatabaseInfo(strPath, params);
1099
1100   if (params.GetArtistId() == -1)
1101     return INFO_ERROR;
1102
1103   CArtist artist;
1104   m_musicDatabase.GetArtistInfo(params.GetArtistId(), artist);
1105
1106   // find album info
1107   ADDON::ScraperPtr scraper;
1108   if (!m_musicDatabase.GetScraperForPath(strPath, scraper, ADDON::ADDON_SCRAPER_ARTISTS) || !scraper)
1109     return INFO_ERROR;
1110   m_musicDatabase.Close();
1111
1112 loop:
1113   CLog::Log(LOGDEBUG, "%s downloading info for: %s", __FUNCTION__, artist.strArtist.c_str());
1114   INFO_RET artistDownloadStatus = DownloadArtistInfo(artist, scraper, artistInfo, pDialog);
1115   if (artistDownloadStatus == INFO_NOT_FOUND)
1116   {
1117     if (pDialog && bAllowSelection)
1118     {
1119       if (!CGUIKeyboardFactory::ShowAndGetInput(artist.strArtist, g_localizeStrings.Get(16025), false))
1120         return INFO_CANCELLED;
1121       goto loop;
1122     }
1123   }
1124   else if (artistDownloadStatus == INFO_ADDED)
1125   {
1126     m_musicDatabase.Open();
1127     m_musicDatabase.SetArtistInfo(params.GetArtistId(), artistInfo.GetArtist());
1128     m_musicDatabase.GetArtistPath(params.GetArtistId(), artist.strPath);
1129     map<string, string> artwork = GetArtistArtwork(artist);
1130     m_musicDatabase.SetArtForItem(params.GetArtistId(), "artist", artwork);
1131     artistInfo.SetLoaded();
1132     m_musicDatabase.Close();
1133   }
1134   return artistDownloadStatus;
1135 }
1136
1137 #define THRESHOLD .95f
1138
1139 INFO_RET CMusicInfoScanner::DownloadAlbumInfo(const CAlbum& album, ADDON::ScraperPtr& info, CMusicAlbumInfo& albumInfo, CGUIDialogProgress* pDialog)
1140 {
1141   if (m_handle)
1142   {
1143     m_handle->SetTitle(StringUtils::Format(g_localizeStrings.Get(20321), info->Name().c_str()));
1144     m_handle->SetText(StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator) + " - " + album.strAlbum);
1145   }
1146
1147   // clear our scraper cache
1148   info->ClearCache();
1149
1150   CMusicInfoScraper scraper(info);
1151   bool bMusicBrainz = false;
1152   if (!album.strMusicBrainzAlbumID.empty())
1153   {
1154     CScraperUrl musicBrainzURL;
1155     if (ResolveMusicBrainz(album.strMusicBrainzAlbumID, info, scraper, musicBrainzURL))
1156     {
1157       CMusicAlbumInfo albumNfo("nfo", musicBrainzURL);
1158       scraper.GetAlbums().clear();
1159       scraper.GetAlbums().push_back(albumNfo);
1160       bMusicBrainz = true;
1161     }
1162   }
1163
1164   // handle nfo files
1165   CStdString strNfo = URIUtils::AddFileToFolder(album.strPath, "album.nfo");
1166   CNfoFile::NFOResult result = CNfoFile::NO_NFO;
1167   CNfoFile nfoReader;
1168   if (XFILE::CFile::Exists(strNfo))
1169   {
1170     CLog::Log(LOGDEBUG,"Found matching nfo file: %s", strNfo.c_str());
1171     result = nfoReader.Create(strNfo, info, -1, album.strPath);
1172     if (result == CNfoFile::FULL_NFO)
1173     {
1174       CLog::Log(LOGDEBUG, "%s Got details from nfo", __FUNCTION__);
1175       nfoReader.GetDetails(albumInfo.GetAlbum());
1176       return INFO_ADDED;
1177     }
1178     else if (result == CNfoFile::URL_NFO || result == CNfoFile::COMBINED_NFO)
1179     {
1180       CScraperUrl scrUrl(nfoReader.ScraperUrl());
1181       CMusicAlbumInfo albumNfo("nfo",scrUrl);
1182       info = nfoReader.GetScraperInfo();
1183       CLog::Log(LOGDEBUG,"-- nfo-scraper: %s",info->Name().c_str());
1184       CLog::Log(LOGDEBUG,"-- nfo url: %s", scrUrl.m_url[0].m_url.c_str());
1185       scraper.SetScraperInfo(info);
1186       scraper.GetAlbums().clear();
1187       scraper.GetAlbums().push_back(albumNfo);
1188     }
1189     else
1190       CLog::Log(LOGERROR,"Unable to find an url in nfo file: %s", strNfo.c_str());
1191   }
1192
1193   if (!scraper.CheckValidOrFallback(CSettings::Get().GetString("musiclibrary.albumsscraper")))
1194   { // the current scraper is invalid, as is the default - bail
1195     CLog::Log(LOGERROR, "%s - current and default scrapers are invalid.  Pick another one", __FUNCTION__);
1196     return INFO_ERROR;
1197   }
1198
1199   if (!scraper.GetAlbumCount())
1200   {
1201     scraper.FindAlbumInfo(album.strAlbum, StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
1202
1203     while (!scraper.Completed())
1204     {
1205       if (m_bStop)
1206       {
1207         scraper.Cancel();
1208         return INFO_CANCELLED;
1209       }
1210       Sleep(1);
1211     }
1212   }
1213
1214   CGUIDialogSelect *pDlg = NULL;
1215   int iSelectedAlbum=0;
1216   if (result == CNfoFile::NO_NFO && !bMusicBrainz)
1217   {
1218     iSelectedAlbum = -1; // set negative so that we can detect a failure
1219     if (scraper.Succeeded() && scraper.GetAlbumCount() >= 1)
1220     {
1221       double bestRelevance = 0;
1222       double minRelevance = THRESHOLD;
1223       if (scraper.GetAlbumCount() > 1) // score the matches
1224       {
1225         //show dialog with all albums found
1226         if (pDialog)
1227         {
1228           pDlg = (CGUIDialogSelect*)g_windowManager.GetWindow(WINDOW_DIALOG_SELECT);
1229           pDlg->SetHeading(g_localizeStrings.Get(181).c_str());
1230           pDlg->Reset();
1231           pDlg->EnableButton(true, 413); // manual
1232         }
1233
1234         for (int i = 0; i < scraper.GetAlbumCount(); ++i)
1235         {
1236           CMusicAlbumInfo& info = scraper.GetAlbum(i);
1237           double relevance = info.GetRelevance();
1238           if (relevance < 0)
1239             relevance = CUtil::AlbumRelevance(info.GetAlbum().strAlbum, album.strAlbum, StringUtils::Join(info.GetAlbum().artist, g_advancedSettings.m_musicItemSeparator), StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
1240
1241           // if we're doing auto-selection (ie querying all albums at once, then allow 95->100% for perfect matches)
1242           // otherwise, perfect matches only
1243           if (relevance >= max(minRelevance, bestRelevance))
1244           { // we auto-select the best of these
1245             bestRelevance = relevance;
1246             iSelectedAlbum = i;
1247           }
1248           if (pDialog)
1249           {
1250             // set the label to [relevance]  album - artist
1251             CStdString strTemp;
1252             strTemp.Format("[%0.2f]  %s", relevance, info.GetTitle2());
1253             CFileItem item(strTemp);
1254             item.m_idepth = i; // use this to hold the index of the album in the scraper
1255             pDlg->Add(&item);
1256           }
1257           if (relevance > .99f) // we're so close, no reason to search further
1258             break;
1259         }
1260
1261         if (pDialog && bestRelevance < THRESHOLD)
1262         {
1263           pDlg->Sort(false);
1264           pDlg->DoModal();
1265
1266           // and wait till user selects one
1267           if (pDlg->GetSelectedLabel() < 0)
1268           { // none chosen
1269             if (!pDlg->IsButtonPressed())
1270               return INFO_CANCELLED;
1271
1272             // manual button pressed
1273             CStdString strNewAlbum = album.strAlbum;
1274             if (!CGUIKeyboardFactory::ShowAndGetInput(strNewAlbum, g_localizeStrings.Get(16011), false)) return INFO_CANCELLED;
1275             if (strNewAlbum == "") return INFO_CANCELLED;
1276
1277             CStdString strNewArtist = StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator);
1278             if (!CGUIKeyboardFactory::ShowAndGetInput(strNewArtist, g_localizeStrings.Get(16025), false)) return INFO_CANCELLED;
1279
1280             pDialog->SetLine(0, strNewAlbum);
1281             pDialog->SetLine(1, strNewArtist);
1282             pDialog->Progress();
1283
1284             CAlbum newAlbum = album;
1285             newAlbum.strAlbum = strNewAlbum;
1286             newAlbum.artist = StringUtils::Split(strNewArtist, g_advancedSettings.m_musicItemSeparator);
1287
1288             return DownloadAlbumInfo(newAlbum, info, albumInfo, pDialog);
1289           }
1290           iSelectedAlbum = pDlg->GetSelectedItem()->m_idepth;
1291         }
1292       }
1293       else
1294       {
1295         CMusicAlbumInfo& info = scraper.GetAlbum(0);
1296         double relevance = info.GetRelevance();
1297         if (relevance < 0)
1298           relevance = CUtil::AlbumRelevance(info.GetAlbum().strAlbum,
1299                                             album.strAlbum,
1300                                             StringUtils::Join(info.GetAlbum().artist, g_advancedSettings.m_musicItemSeparator),
1301                                             StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
1302         if (relevance < THRESHOLD)
1303           return INFO_NOT_FOUND;
1304
1305         iSelectedAlbum = 0;
1306       }
1307     }
1308
1309     if (iSelectedAlbum < 0)
1310       return INFO_NOT_FOUND;
1311
1312   }
1313
1314   scraper.LoadAlbumInfo(iSelectedAlbum);
1315   while (!scraper.Completed())
1316   {
1317     if (m_bStop)
1318     {
1319       scraper.Cancel();
1320       return INFO_CANCELLED;
1321     }
1322     Sleep(1);
1323   }
1324
1325   if (!scraper.Succeeded())
1326     return INFO_ERROR;
1327
1328   albumInfo = scraper.GetAlbum(iSelectedAlbum);
1329
1330   if (result == CNfoFile::COMBINED_NFO)
1331     nfoReader.GetDetails(albumInfo.GetAlbum(), NULL, true);
1332
1333   return INFO_ADDED;
1334 }
1335
1336 void CMusicInfoScanner::GetAlbumArtwork(long id, const CAlbum &album)
1337 {
1338   if (album.thumbURL.m_url.size())
1339   {
1340     if (m_musicDatabase.GetArtForItem(id, "album", "thumb").empty())
1341     {
1342       string thumb = CScraperUrl::GetThumbURL(album.thumbURL.GetFirstThumb());
1343       if (!thumb.empty())
1344       {
1345         CTextureCache::Get().BackgroundCacheImage(thumb);
1346         m_musicDatabase.SetArtForItem(id, "album", "thumb", thumb);
1347       }
1348     }
1349   }
1350 }
1351
1352 INFO_RET CMusicInfoScanner::DownloadArtistInfo(const CArtist& artist, ADDON::ScraperPtr& info, MUSIC_GRABBER::CMusicArtistInfo& artistInfo, CGUIDialogProgress* pDialog)
1353 {
1354   if (m_handle)
1355   {
1356     m_handle->SetTitle(StringUtils::Format(g_localizeStrings.Get(20320), info->Name().c_str()));
1357     m_handle->SetText(artist.strArtist);
1358   }
1359
1360   // clear our scraper cache
1361   info->ClearCache();
1362
1363   CMusicInfoScraper scraper(info);
1364   bool bMusicBrainz = false;
1365   if (!artist.strMusicBrainzArtistID.empty())
1366   {
1367     CScraperUrl musicBrainzURL;
1368     if (ResolveMusicBrainz(artist.strMusicBrainzArtistID, info, scraper, musicBrainzURL))
1369     {
1370       CMusicArtistInfo artistNfo("nfo", musicBrainzURL);
1371       scraper.GetArtists().clear();
1372       scraper.GetArtists().push_back(artistNfo);
1373       bMusicBrainz = true;
1374     }
1375   }
1376
1377   // handle nfo files
1378   CStdString strNfo = URIUtils::AddFileToFolder(artist.strPath, "artist.nfo");
1379   CNfoFile::NFOResult result=CNfoFile::NO_NFO;
1380   CNfoFile nfoReader;
1381   if (XFILE::CFile::Exists(strNfo))
1382   {
1383     CLog::Log(LOGDEBUG,"Found matching nfo file: %s", strNfo.c_str());
1384     result = nfoReader.Create(strNfo, info);
1385     if (result == CNfoFile::FULL_NFO)
1386     {
1387       CLog::Log(LOGDEBUG, "%s Got details from nfo", __FUNCTION__);
1388       nfoReader.GetDetails(artistInfo.GetArtist());
1389       return INFO_ADDED;
1390     }
1391     else if (result == CNfoFile::URL_NFO || result == CNfoFile::COMBINED_NFO)
1392     {
1393       CScraperUrl scrUrl(nfoReader.ScraperUrl());
1394       CMusicArtistInfo artistNfo("nfo",scrUrl);
1395       info = nfoReader.GetScraperInfo();
1396       CLog::Log(LOGDEBUG,"-- nfo-scraper: %s",info->Name().c_str());
1397       CLog::Log(LOGDEBUG,"-- nfo url: %s", scrUrl.m_url[0].m_url.c_str());
1398       scraper.SetScraperInfo(info);
1399       scraper.GetArtists().push_back(artistNfo);
1400     }
1401     else
1402       CLog::Log(LOGERROR,"Unable to find an url in nfo file: %s", strNfo.c_str());
1403   }
1404
1405   if (!scraper.GetArtistCount())
1406   {
1407     scraper.FindArtistInfo(artist.strArtist);
1408
1409     while (!scraper.Completed())
1410     {
1411       if (m_bStop)
1412       {
1413         scraper.Cancel();
1414         return INFO_CANCELLED;
1415       }
1416       Sleep(1);
1417     }
1418   }
1419
1420   int iSelectedArtist = 0;
1421   if (result == CNfoFile::NO_NFO && !bMusicBrainz)
1422   {
1423     if (scraper.GetArtistCount() >= 1)
1424     {
1425       // now load the first match
1426       if (pDialog && scraper.GetArtistCount() > 1)
1427       {
1428         // if we found more then 1 album, let user choose one
1429         CGUIDialogSelect *pDlg = (CGUIDialogSelect*)g_windowManager.GetWindow(WINDOW_DIALOG_SELECT);
1430         if (pDlg)
1431         {
1432           pDlg->SetHeading(g_localizeStrings.Get(21890));
1433           pDlg->Reset();
1434           pDlg->EnableButton(true, 413); // manual
1435
1436           for (int i = 0; i < scraper.GetArtistCount(); ++i)
1437           {
1438             // set the label to artist
1439             CFileItem item(scraper.GetArtist(i).GetArtist());
1440             CStdString strTemp=scraper.GetArtist(i).GetArtist().strArtist;
1441             if (!scraper.GetArtist(i).GetArtist().strBorn.IsEmpty())
1442               strTemp += " ("+scraper.GetArtist(i).GetArtist().strBorn+")";
1443             if (!scraper.GetArtist(i).GetArtist().genre.empty())
1444             {
1445               CStdString genres = StringUtils::Join(scraper.GetArtist(i).GetArtist().genre, g_advancedSettings.m_musicItemSeparator);
1446               if (!genres.empty())
1447                 strTemp.Format("[%s] %s", genres.c_str(), strTemp.c_str());
1448             }
1449             item.SetLabel(strTemp);
1450             item.m_idepth = i; // use this to hold the index of the album in the scraper
1451             pDlg->Add(&item);
1452           }
1453           pDlg->DoModal();
1454
1455           // and wait till user selects one
1456           if (pDlg->GetSelectedLabel() < 0)
1457           { // none chosen
1458             if (!pDlg->IsButtonPressed())
1459               return INFO_CANCELLED;
1460
1461             // manual button pressed
1462             CStdString strNewArtist = artist.strArtist;
1463             if (!CGUIKeyboardFactory::ShowAndGetInput(strNewArtist, g_localizeStrings.Get(16025), false)) return INFO_CANCELLED;
1464
1465             if (pDialog)
1466             {
1467               pDialog->SetLine(0, strNewArtist);
1468               pDialog->Progress();
1469             }
1470
1471             CArtist newArtist;
1472             newArtist.strArtist = strNewArtist;
1473             return DownloadArtistInfo(newArtist, info, artistInfo, pDialog);
1474           }
1475           iSelectedArtist = pDlg->GetSelectedItem()->m_idepth;
1476         }
1477       }
1478     }
1479     else
1480       return INFO_NOT_FOUND;
1481   }
1482
1483   scraper.LoadArtistInfo(iSelectedArtist, artist.strArtist);
1484   while (!scraper.Completed())
1485   {
1486     if (m_bStop)
1487     {
1488       scraper.Cancel();
1489       return INFO_CANCELLED;
1490     }
1491     Sleep(1);
1492   }
1493
1494   if (!scraper.Succeeded())
1495     return INFO_ERROR;
1496
1497   artistInfo = scraper.GetArtist(iSelectedArtist);
1498
1499   if (result == CNfoFile::COMBINED_NFO)
1500     nfoReader.GetDetails(artistInfo.GetArtist(), NULL, true);
1501
1502   return INFO_ADDED;
1503 }
1504
1505 bool CMusicInfoScanner::ResolveMusicBrainz(const CStdString strMusicBrainzID, ScraperPtr &preferredScraper, CMusicInfoScraper &musicInfoScraper, CScraperUrl &musicBrainzURL)
1506 {
1507   // We have a MusicBrainz ID
1508   // Get a scraper that can resolve it to a MusicBrainz URL & force our
1509   // search directly to the specific album.
1510   bool bMusicBrainz = false;
1511   ADDON::TYPE type = ScraperTypeFromContent(preferredScraper->Content());
1512
1513   CFileItemList items;
1514   ADDON::AddonPtr addon;
1515   ADDON::ScraperPtr defaultScraper;
1516   if (ADDON::CAddonMgr::Get().GetDefault(type, addon))
1517     defaultScraper = boost::dynamic_pointer_cast<CScraper>(addon);
1518
1519   vector<ScraperPtr> vecScrapers;
1520
1521   // add selected scraper - first proirity
1522   if (preferredScraper)
1523     vecScrapers.push_back(preferredScraper);
1524
1525   // Add all scrapers except selected and default
1526   VECADDONS addons;
1527   CAddonMgr::Get().GetAddons(type, addons);
1528
1529   for (unsigned i = 0; i < addons.size(); ++i)
1530   {
1531     ScraperPtr scraper = boost::dynamic_pointer_cast<CScraper>(addons[i]);
1532
1533     // skip if scraper requires settings and there's nothing set yet
1534     if (!scraper || (scraper->RequiresSettings() && !scraper->HasUserSettings()))
1535       continue;
1536
1537     if((!preferredScraper || preferredScraper->ID() != scraper->ID()) && (!defaultScraper || defaultScraper->ID() != scraper->ID()) )
1538       vecScrapers.push_back(scraper);
1539   }
1540
1541   // add default scraper - not user selectable so it's last priority
1542   if(defaultScraper &&
1543      (!preferredScraper || preferredScraper->ID() != defaultScraper->ID()) &&
1544      (!defaultScraper->RequiresSettings() || defaultScraper->HasUserSettings()))
1545     vecScrapers.push_back(defaultScraper);
1546
1547   for (unsigned int i=0; i < vecScrapers.size(); ++i)
1548   {
1549     if (vecScrapers[i]->Type() != type)
1550       continue;
1551
1552     vecScrapers[i]->ClearCache();
1553     try
1554     {
1555       musicBrainzURL = vecScrapers[i]->ResolveIDToUrl(strMusicBrainzID);
1556     }
1557     catch (const ADDON::CScraperError &sce)
1558     {
1559       if (!sce.FAborted())
1560         continue;
1561     }
1562     if (!musicBrainzURL.m_url.empty())
1563     {
1564       Sleep(2000); // MusicBrainz rate-limits queries to 1 p.s - once we hit the rate-limiter
1565                    // they start serving up the 'you hit the rate-limiter' page fast - meaning
1566                    // we will never get below the rate-limit threshold again in a specific run.
1567                    // This helps us to avoidthe rate-limiter as far as possible.
1568       CLog::Log(LOGDEBUG,"-- nfo-scraper: %s",vecScrapers[i]->Name().c_str());
1569       CLog::Log(LOGDEBUG,"-- nfo url: %s", musicBrainzURL.m_url[0].m_url.c_str());
1570       musicInfoScraper.SetScraperInfo(vecScrapers[i]);
1571       bMusicBrainz = true;
1572       break;
1573     }
1574   }
1575
1576   return bMusicBrainz;
1577 }
1578
1579 map<string, string> CMusicInfoScanner::GetArtistArtwork(const CArtist& artist)
1580 {
1581   map<string, string> artwork;
1582
1583   // check thumb
1584   CStdString strFolder;
1585   CStdString thumb;
1586   if (!artist.strPath.IsEmpty())
1587   {
1588     strFolder = artist.strPath;
1589     for (int i = 0; i < 3 && thumb.IsEmpty(); ++i)
1590     {
1591       CFileItem item(strFolder, true);
1592       thumb = item.GetUserMusicThumb(true);
1593       strFolder = URIUtils::GetParentPath(strFolder);
1594     }
1595   }
1596   if (thumb.IsEmpty())
1597     thumb = CScraperUrl::GetThumbURL(artist.thumbURL.GetFirstThumb());
1598   if (!thumb.IsEmpty())
1599   {
1600     CTextureCache::Get().BackgroundCacheImage(thumb);
1601     artwork.insert(make_pair("thumb", thumb));
1602   }
1603
1604   // check fanart
1605   CStdString fanart;
1606   if (!artist.strPath.IsEmpty())
1607   {
1608     strFolder = artist.strPath;
1609     for (int i = 0; i < 3 && fanart.IsEmpty(); ++i)
1610     {
1611       CFileItem item(strFolder, true);
1612       fanart = item.GetLocalFanart();
1613       strFolder = URIUtils::GetParentPath(strFolder);
1614     }
1615   }
1616   if (fanart.IsEmpty())
1617     fanart = artist.fanart.GetImageURL();
1618   if (!fanart.IsEmpty())
1619   {
1620     CTextureCache::Get().BackgroundCacheImage(fanart);
1621     artwork.insert(make_pair("fanart", fanart));
1622   }
1623
1624   return artwork;
1625 }
1626
1627 // This function is the Run() function of the IRunnable
1628 // CFileCountReader and runs in a separate thread.
1629 void CMusicInfoScanner::Run()
1630 {
1631   int count = 0;
1632   for (set<std::string>::iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end() && !m_bStop; ++it)
1633   {
1634     count+=CountFilesRecursively(*it);
1635   }
1636   m_itemCount = count;
1637 }
1638
1639 // Recurse through all folders we scan and count files
1640 int CMusicInfoScanner::CountFilesRecursively(const CStdString& strPath)
1641 {
1642   // load subfolder
1643   CFileItemList items;
1644   CDirectory::GetDirectory(strPath, items, g_advancedSettings.m_musicExtensions, DIR_FLAG_NO_FILE_DIRS);
1645
1646   if (m_bStop)
1647     return 0;
1648
1649   // true for recursive counting
1650   int count = CountFiles(items, true);
1651   return count;
1652 }
1653
1654 int CMusicInfoScanner::CountFiles(const CFileItemList &items, bool recursive)
1655 {
1656   int count = 0;
1657   for (int i=0; i<items.Size(); ++i)
1658   {
1659     const CFileItemPtr pItem=items[i];
1660
1661     if (recursive && pItem->m_bIsFolder)
1662       count+=CountFilesRecursively(pItem->GetPath());
1663     else if (pItem->IsAudio() && !pItem->IsPlayList() && !pItem->IsNFO())
1664       count++;
1665   }
1666   return count;
1667 }