code.vuplus.com Git - vuplus_xbmc/blob - xbmc/music/infoscanner/MusicInfoScanner.cpp

   1 /*
   2  *      Copyright (C) 2005-2013 Team XBMC
   3  *      http://www.xbmc.org
   4  *
   5  *  This Program is free software; you can redistribute it and/or modify
   6  *  it under the terms of the GNU General Public License as published by
   7  *  the Free Software Foundation; either version 2, or (at your option)
   8  *  any later version.
   9  *
  10  *  This Program is distributed in the hope that it will be useful,
  11  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  *  GNU General Public License for more details.
  14  *
  15  *  You should have received a copy of the GNU General Public License
  16  *  along with XBMC; see the file COPYING.  If not, see
  17  *  <http://www.gnu.org/licenses/>.
  18  *
  19  */
  20
  21 #include "threads/SystemClock.h"
  22 #include "MusicInfoScanner.h"
  23 #include "music/tags/MusicInfoTagLoaderFactory.h"
  24 #include "MusicAlbumInfo.h"
  25 #include "MusicInfoScraper.h"
  26 #include "filesystem/MusicDatabaseDirectory.h"
  27 #include "filesystem/MusicDatabaseDirectory/DirectoryNode.h"
  28 #include "Util.h"
  29 #include "utils/md5.h"
  30 #include "GUIInfoManager.h"
  31 #include "utils/Variant.h"
  32 #include "NfoFile.h"
  33 #include "music/tags/MusicInfoTag.h"
  34 #include "guilib/GUIWindowManager.h"
  35 #include "dialogs/GUIDialogExtendedProgressBar.h"
  36 #include "dialogs/GUIDialogProgress.h"
  37 #include "dialogs/GUIDialogSelect.h"
  38 #include "guilib/GUIKeyboardFactory.h"
  39 #include "filesystem/File.h"
  40 #include "filesystem/Directory.h"
  41 #include "settings/AdvancedSettings.h"
  42 #include "settings/Settings.h"
  43 #include "FileItem.h"
  44 #include "guilib/LocalizeStrings.h"
  45 #include "utils/StringUtils.h"
  46 #include "utils/TimeUtils.h"
  47 #include "utils/log.h"
  48 #include "utils/URIUtils.h"
  49 #include "TextureCache.h"
  50 #include "music/MusicThumbLoader.h"
  51 #include "interfaces/AnnouncementManager.h"
  52 #include "GUIUserMessages.h"
  53 #include "addons/AddonManager.h"
  54 #include "addons/Scraper.h"
  55
  56 #include <algorithm>
  57
  58 using namespace std;
  59 using namespace MUSIC_INFO;
  60 using namespace XFILE;
  61 using namespace MUSICDATABASEDIRECTORY;
  62 using namespace MUSIC_GRABBER;
  63 using namespace ADDON;
  64
  65 CMusicInfoScanner::CMusicInfoScanner() : CThread("MusicInfoScanner"), m_fileCountReader(this, "MusicFileCounter")
  66 {
  67   m_bRunning = false;
  68   m_showDialog = false;
  69   m_handle = NULL;
  70   m_bCanInterrupt = false;
  71   m_currentItem=0;
  72   m_itemCount=0;
  73   m_flags = 0;
  74 }
  75
  76 CMusicInfoScanner::~CMusicInfoScanner()
  77 {
  78 }
  79
  80 void CMusicInfoScanner::Process()
  81 {
  82   ANNOUNCEMENT::CAnnouncementManager::Announce(ANNOUNCEMENT::AudioLibrary, "xbmc", "OnScanStarted");
  83   try
  84   {
  85     unsigned int tick = XbmcThreads::SystemClockMillis();
  86
  87     m_musicDatabase.Open();
  88
  89     if (m_showDialog && !CSettings::Get().GetBool("musiclibrary.backgroundupdate"))
  90     {
  91       CGUIDialogExtendedProgressBar* dialog =
  92         (CGUIDialogExtendedProgressBar*)g_windowManager.GetWindow(WINDOW_DIALOG_EXT_PROGRESS);
  93       m_handle = dialog->GetHandle(g_localizeStrings.Get(314));
  94     }
  95
  96     m_bCanInterrupt = true;
  97
  98     if (m_scanType == 0) // load info from files
  99     {
 100       CLog::Log(LOGDEBUG, "%s - Starting scan", __FUNCTION__);
 101
 102       if (m_handle)
 103         m_handle->SetTitle(g_localizeStrings.Get(505));
 104
 105       // Reset progress vars
 106       m_currentItem=0;
 107       m_itemCount=-1;
 108
 109       // Create the thread to count all files to be scanned
 110       SetPriority( GetMinPriority() );
 111       if (m_handle)
 112         m_fileCountReader.Create();
 113
 114       // Database operations should not be canceled
 115       // using Interupt() while scanning as it could
 116       // result in unexpected behaviour.
 117       m_bCanInterrupt = false;
 118       m_needsCleanup = false;
 119
 120       bool commit = false;
 121       bool cancelled = false;
 122       for (std::set<std::string>::const_iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end(); it++)
 123       {
 124         /*
 125          * A copy of the directory path is used because the path supplied is
 126          * immediately removed from the m_pathsToScan set in DoScan(). If the
 127          * reference points to the entry in the set a null reference error
 128          * occurs.
 129          */
 130         if (!DoScan(*it))
 131           cancelled = true;
 132         commit = !cancelled;
 133       }
 134
 135       if (commit)
 136       {
 137         g_infoManager.ResetLibraryBools();
 138
 139         if (m_needsCleanup)
 140         {
 141           if (m_handle)
 142           {
 143             m_handle->SetTitle(g_localizeStrings.Get(700));
 144             m_handle->SetText("");
 145           }
 146
 147           m_musicDatabase.CleanupOrphanedItems();
 148
 149           if (m_handle)
 150             m_handle->SetTitle(g_localizeStrings.Get(331));
 151
 152           m_musicDatabase.Compress(false);
 153         }
 154       }
 155
 156       m_fileCountReader.StopThread();
 157
 158       m_musicDatabase.EmptyCache();
 159
 160       tick = XbmcThreads::SystemClockMillis() - tick;
 161       CLog::Log(LOGNOTICE, "My Music: Scanning for music info using worker thread, operation took %s", StringUtils::SecondsToTimeString(tick / 1000).c_str());
 162     }
 163     if (m_scanType == 1) // load album info
 164     {
 165       for (std::set<std::string>::const_iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end(); ++it)
 166       {
 167         CQueryParams params;
 168         CDirectoryNode::GetDatabaseInfo(*it, params);
 169         if (m_musicDatabase.HasAlbumInfo(params.GetArtistId())) // should this be here?
 170           continue;
 171
 172         CAlbum album;
 173         m_musicDatabase.GetAlbumInfo(params.GetAlbumId(), album, &album.songs);
 174         if (m_handle)
 175         {
 176           float percentage = (float) std::distance(it, m_pathsToScan.end()) / m_pathsToScan.size();
 177           m_handle->SetText(StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator) + " - " + album.strAlbum);
 178           m_handle->SetPercentage(percentage);
 179         }
 180
 181         CMusicAlbumInfo albumInfo;
 182         UpdateDatabaseAlbumInfo(*it, albumInfo, false);
 183
 184         if (m_bStop)
 185           break;
 186       }
 187     }
 188     if (m_scanType == 2) // load artist info
 189     {
 190       for (std::set<std::string>::const_iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end(); ++it)
 191       {
 192         CQueryParams params;
 193         CDirectoryNode::GetDatabaseInfo(*it, params);
 194         if (m_musicDatabase.HasArtistInfo(params.GetArtistId())) // should this be here?
 195             continue;
 196
 197         CArtist artist;
 198         m_musicDatabase.GetArtistInfo(params.GetArtistId(), artist);
 199         m_musicDatabase.GetArtistPath(params.GetArtistId(), artist.strPath);
 200
 201         if (m_handle)
 202         {
 203           float percentage = (float) (std::distance(m_pathsToScan.begin(), it) / m_pathsToScan.size()) * 100;
 204           m_handle->SetText(artist.strArtist);
 205           m_handle->SetPercentage(percentage);
 206         }
 207
 208         CMusicArtistInfo artistInfo;
 209         UpdateDatabaseArtistInfo(*it, artistInfo, false);
 210
 211         if (m_bStop)
 212           break;
 213       }
 214     }
 215
 216   }
 217   catch (...)
 218   {
 219     CLog::Log(LOGERROR, "MusicInfoScanner: Exception while scanning.");
 220   }
 221   m_musicDatabase.Close();
 222   CLog::Log(LOGDEBUG, "%s - Finished scan", __FUNCTION__);
 223
 224   m_bRunning = false;
 225   ANNOUNCEMENT::CAnnouncementManager::Announce(ANNOUNCEMENT::AudioLibrary, "xbmc", "OnScanFinished");
 226
 227   // we need to clear the musicdb cache and update any active lists
 228   CUtil::DeleteMusicDatabaseDirectoryCache();
 229   CGUIMessage msg(GUI_MSG_SCAN_FINISHED, 0, 0, 0);
 230   g_windowManager.SendThreadMessage(msg);
 231
 232   if (m_handle)
 233     m_handle->MarkFinished();
 234   m_handle = NULL;
 235 }
 236
 237 void CMusicInfoScanner::Start(const CStdString& strDirectory, int flags)
 238 {
 239   m_fileCountReader.StopThread();
 240   StopThread();
 241   m_pathsToScan.clear();
 242   m_flags = flags;
 243
 244   if (strDirectory.IsEmpty())
 245   { // scan all paths in the database.  We do this by scanning all paths in the db, and crossing them off the list as
 246     // we go.
 247     m_musicDatabase.Open();
 248     m_musicDatabase.GetPaths(m_pathsToScan);
 249     m_musicDatabase.Close();
 250   }
 251   else
 252     m_pathsToScan.insert(strDirectory);
 253
 254   m_scanType = 0;
 255   Create();
 256   m_bRunning = true;
 257 }
 258
 259 void CMusicInfoScanner::FetchAlbumInfo(const CStdString& strDirectory,
 260                                        bool refresh)
 261 {
 262   m_fileCountReader.StopThread();
 263   StopThread();
 264   m_pathsToScan.clear();
 265
 266   CFileItemList items;
 267   if (strDirectory.IsEmpty())
 268   {
 269     m_musicDatabase.Open();
 270     m_musicDatabase.GetAlbumsNav("musicdb://albums/", items);
 271     m_musicDatabase.Close();
 272   }
 273   else
 274   {
 275     if (URIUtils::HasSlashAtEnd(strDirectory)) // directory
 276       CDirectory::GetDirectory(strDirectory,items);
 277     else
 278     {
 279       CFileItemPtr item(new CFileItem(strDirectory,false));
 280       items.Add(item);
 281     }
 282   }
 283
 284   m_musicDatabase.Open();
 285   for (int i=0;i<items.Size();++i)
 286   {
 287     if (CMusicDatabaseDirectory::IsAllItem(items[i]->GetPath()) || items[i]->IsParentFolder())
 288       continue;
 289
 290     m_pathsToScan.insert(items[i]->GetPath());
 291     if (refresh)
 292     {
 293       m_musicDatabase.DeleteAlbumInfo(items[i]->GetMusicInfoTag()->GetDatabaseId());
 294     }
 295   }
 296   m_musicDatabase.Close();
 297
 298   m_scanType = 1;
 299   Create();
 300   m_bRunning = true;
 301 }
 302
 303 void CMusicInfoScanner::FetchArtistInfo(const CStdString& strDirectory,
 304                                         bool refresh)
 305 {
 306   m_fileCountReader.StopThread();
 307   StopThread();
 308   m_pathsToScan.clear();
 309   CFileItemList items;
 310
 311   if (strDirectory.IsEmpty())
 312   {
 313     m_musicDatabase.Open();
 314     m_musicDatabase.GetArtistsNav("musicdb://artists/", items, false, -1);
 315     m_musicDatabase.Close();
 316   }
 317   else
 318   {
 319     if (URIUtils::HasSlashAtEnd(strDirectory)) // directory
 320       CDirectory::GetDirectory(strDirectory,items);
 321     else
 322     {
 323       CFileItemPtr newItem(new CFileItem(strDirectory,false));
 324       items.Add(newItem);
 325     }
 326   }
 327
 328   m_musicDatabase.Open();
 329   for (int i=0;i<items.Size();++i)
 330   {
 331     if (CMusicDatabaseDirectory::IsAllItem(items[i]->GetPath()) || items[i]->IsParentFolder())
 332       continue;
 333
 334     m_pathsToScan.insert(items[i]->GetPath());
 335     if (refresh)
 336     {
 337       m_musicDatabase.DeleteArtistInfo(items[i]->GetMusicInfoTag()->GetDatabaseId());
 338     }
 339   }
 340   m_musicDatabase.Close();
 341
 342   m_scanType = 2;
 343   Create();
 344   m_bRunning = true;
 345 }
 346
 347 bool CMusicInfoScanner::IsScanning()
 348 {
 349   return m_bRunning;
 350 }
 351
 352 void CMusicInfoScanner::Stop()
 353 {
 354   if (m_bCanInterrupt)
 355     m_musicDatabase.Interupt();
 356
 357   StopThread(false);
 358 }
 359
 360 static void OnDirectoryScanned(const CStdString& strDirectory)
 361 {
 362   CGUIMessage msg(GUI_MSG_DIRECTORY_SCANNED, 0, 0, 0);
 363   msg.SetStringParam(strDirectory);
 364   g_windowManager.SendThreadMessage(msg);
 365 }
 366
 367 static CStdString Prettify(const CStdString& strDirectory)
 368 {
 369   CURL url(strDirectory);
 370   CStdString strStrippedPath = url.GetWithoutUserDetails();
 371   CURL::Decode(strStrippedPath);
 372
 373   return strStrippedPath;
 374 }
 375
 376 bool CMusicInfoScanner::DoScan(const CStdString& strDirectory)
 377 {
 378   if (m_handle)
 379     m_handle->SetText(Prettify(strDirectory));
 380
 381   // Discard all excluded files defined by m_musicExcludeRegExps
 382   CStdStringArray regexps = g_advancedSettings.m_audioExcludeFromScanRegExps;
 383   if (CUtil::ExcludeFileOrFolder(strDirectory, regexps))
 384     return true;
 385
 386   // load subfolder
 387   CFileItemList items;
 388   CDirectory::GetDirectory(strDirectory, items, g_advancedSettings.m_musicExtensions + "|.jpg|.tbn|.lrc|.cdg");
 389
 390   // sort and get the path hash.  Note that we don't filter .cue sheet items here as we want
 391   // to detect changes in the .cue sheet as well.  The .cue sheet items only need filtering
 392   // if we have a changed hash.
 393   items.Sort(SORT_METHOD_LABEL, SortOrderAscending);
 394   CStdString hash;
 395   GetPathHash(items, hash);
 396
 397   // check whether we need to rescan or not
 398   CStdString dbHash;
 399   if ((m_flags & SCAN_RESCAN) || !m_musicDatabase.GetPathHash(strDirectory, dbHash) || dbHash != hash)
 400   { // path has changed - rescan
 401     if (dbHash.IsEmpty())
 402       CLog::Log(LOGDEBUG, "%s Scanning dir '%s' as not in the database", __FUNCTION__, strDirectory.c_str());
 403     else
 404       CLog::Log(LOGDEBUG, "%s Rescanning dir '%s' due to change", __FUNCTION__, strDirectory.c_str());
 405
 406     // filter items in the sub dir (for .cue sheet support)
 407     items.FilterCueItems();
 408     items.Sort(SORT_METHOD_LABEL, SortOrderAscending);
 409
 410     // and then scan in the new information
 411     if (RetrieveMusicInfo(strDirectory, items) > 0)
 412     {
 413       if (m_handle)
 414         OnDirectoryScanned(strDirectory);
 415     }
 416
 417     // save information about this folder
 418     m_musicDatabase.SetPathHash(strDirectory, hash);
 419   }
 420   else
 421   { // path is the same - no need to rescan
 422     CLog::Log(LOGDEBUG, "%s Skipping dir '%s' due to no change", __FUNCTION__, strDirectory.c_str());
 423     m_currentItem += CountFiles(items, false);  // false for non-recursive
 424
 425     // updated the dialog with our progress
 426     if (m_handle)
 427     {
 428       if (m_itemCount>0)
 429         m_handle->SetPercentage(m_currentItem/(float)m_itemCount*100);
 430       OnDirectoryScanned(strDirectory);
 431     }
 432   }
 433
 434   // now scan the subfolders
 435   for (int i = 0; i < items.Size(); ++i)
 436   {
 437     CFileItemPtr pItem = items[i];
 438
 439     if (m_bStop)
 440       break;
 441     // if we have a directory item (non-playlist) we then recurse into that folder
 442     if (pItem->m_bIsFolder && !pItem->IsParentFolder() && !pItem->IsPlayList())
 443     {
 444       CStdString strPath=pItem->GetPath();
 445       if (!DoScan(strPath))
 446       {
 447         m_bStop = true;
 448       }
 449     }
 450   }
 451
 452   return !m_bStop;
 453 }
 454
 455 INFO_RET CMusicInfoScanner::ScanTags(const CFileItemList& items, CFileItemList& scannedItems)
 456 {
 457   CStdStringArray regexps = g_advancedSettings.m_audioExcludeFromScanRegExps;
 458
 459   for (int i = 0; i < items.Size(); ++i)
 460   {
 461     if (m_bStop)
 462       return INFO_CANCELLED;
 463
 464     CFileItemPtr pItem = items[i];
 465
 466     if (CUtil::ExcludeFileOrFolder(pItem->GetPath(), regexps))
 467       continue;
 468
 469     if (pItem->m_bIsFolder || pItem->IsPlayList() || pItem->IsPicture() || pItem->IsLyrics())
 470       continue;
 471
 472     m_currentItem++;
 473
 474     CMusicInfoTag& tag = *pItem->GetMusicInfoTag();
 475     if (!tag.Loaded())
 476     {
 477       auto_ptr<IMusicInfoTagLoader> pLoader (CMusicInfoTagLoaderFactory::CreateLoader(pItem->GetPath()));
 478       if (NULL != pLoader.get())
 479         pLoader->Load(pItem->GetPath(), tag);
 480     }
 481
 482     if (m_handle && m_itemCount>0)
 483       m_handle->SetPercentage(m_currentItem/(float)m_itemCount*100);
 484
 485     if (!tag.Loaded())
 486     {
 487       CLog::Log(LOGDEBUG, "%s - No tag found for: %s", __FUNCTION__, pItem->GetPath().c_str());
 488       continue;
 489     }
 490     scannedItems.Add(pItem);
 491   }
 492   return INFO_ADDED;
 493 }
 494
 495 void CMusicInfoScanner::FileItemsToAlbums(CFileItemList& items, VECALBUMS& albums, MAPSONGS* songsMap /* = NULL */)
 496 {
 497   for (int i = 0; i < items.Size(); ++i)
 498   {
 499     CFileItemPtr pItem = items[i];
 500     CMusicInfoTag& tag = *pItem->GetMusicInfoTag();
 501     CSong song(*pItem);
 502
 503     // keep the db-only fields intact on rescan...
 504     if (songsMap != NULL)
 505     {
 506       MAPSONGS::iterator it = songsMap->find(pItem->GetPath());
 507       if (it != songsMap->end())
 508       {
 509         song.iTimesPlayed = it->second.iTimesPlayed;
 510         song.lastPlayed = it->second.lastPlayed;
 511         song.iKaraokeNumber = it->second.iKaraokeNumber;
 512         if (song.rating == '0')    song.rating = it->second.rating;
 513         if (song.strThumb.empty()) song.strThumb = it->second.strThumb;
 514       }
 515     }
 516
 517     if (!tag.GetMusicBrainzArtistID().empty())
 518     {
 519       for (vector<string>::const_iterator it = tag.GetMusicBrainzArtistID().begin(); it != tag.GetMusicBrainzArtistID().end(); ++it)
 520       {
 521         CStdString strJoinPhrase = (it == --tag.GetMusicBrainzArtistID().end() ? "" : g_advancedSettings.m_musicItemSeparator);
 522         CArtistCredit mbartist(tag.GetArtist().empty() ? "" : tag.GetArtist()[0], *it, strJoinPhrase);
 523         song.artistCredits.push_back(mbartist);
 524       }
 525       song.artist = tag.GetArtist();
 526     }
 527     else
 528     {
 529       for (vector<string>::const_iterator it = tag.GetArtist().begin(); it != tag.GetArtist().end(); ++it)
 530       {
 531         CStdString strJoinPhrase = (it == --tag.GetArtist().end() ? "" : g_advancedSettings.m_musicItemSeparator);
 532         CArtistCredit nonmbartist(*it, strJoinPhrase);
 533         song.artistCredits.push_back(nonmbartist);
 534       }
 535       song.artist = tag.GetArtist();
 536     }
 537
 538     bool found = false;
 539     for (VECALBUMS::iterator it = albums.begin(); it != albums.end(); ++it)
 540     {
 541       if (it->strAlbum == tag.GetAlbum() && it->strMusicBrainzAlbumID == tag.GetMusicBrainzAlbumID())
 542       {
 543         it->songs.push_back(song);
 544         found = true;
 545       }
 546     }
 547     if (!found)
 548     {
 549       CAlbum album(*pItem.get());
 550       if (!tag.GetMusicBrainzAlbumArtistID().empty())
 551       {
 552         for (vector<string>::const_iterator it = tag.GetMusicBrainzAlbumArtistID().begin(); it != tag.GetMusicBrainzAlbumArtistID().end(); ++it)
 553         {
 554           // Picard always stored the display artist string in the first artist slot, no need to split it
 555           CStdString strJoinPhrase = (it == --tag.GetMusicBrainzAlbumArtistID().end() ? "" : g_advancedSettings.m_musicItemSeparator);
 556           CArtistCredit mbartist(tag.GetAlbumArtist().empty() ? "" : tag.GetAlbumArtist()[0], *it, strJoinPhrase);
 557           album.artistCredits.push_back(mbartist);
 558         }
 559         album.artist = tag.GetAlbumArtist();
 560       }
 561       else
 562       {
 563         for (vector<string>::const_iterator it = tag.GetAlbumArtist().begin(); it != tag.GetAlbumArtist().end(); ++it)
 564         {
 565           CStdString strJoinPhrase = (it == --tag.GetAlbumArtist().end() ? "" : g_advancedSettings.m_musicItemSeparator);
 566           CArtistCredit nonmbartist(*it, strJoinPhrase);
 567           album.artistCredits.push_back(nonmbartist);
 568         }
 569         album.artist = tag.GetAlbumArtist();
 570       }
 571       album.songs.push_back(song);
 572       albums.push_back(album);
 573     }
 574   }
 575 }
 576
 577 int CMusicInfoScanner::RetrieveMusicInfo(const CStdString& strDirectory, CFileItemList& items)
 578 {
 579   MAPSONGS songsMap;
 580
 581   // get all information for all files in current directory from database, and remove them
 582   if (m_musicDatabase.RemoveSongsFromPath(strDirectory, songsMap))
 583     m_needsCleanup = true;
 584
 585   CFileItemList scannedItems;
 586   if (ScanTags(items, scannedItems) == INFO_CANCELLED)
 587     return 0;
 588
 589   VECALBUMS albums;
 590   FileItemsToAlbums(scannedItems, albums, &songsMap);
 591   if (!(m_flags & SCAN_ONLINE))
 592     FixupAlbums(albums);
 593   FindArtForAlbums(albums, items.GetPath());
 594
 595   int numAdded = 0;
 596   ADDON::AddonPtr addon;
 597   ADDON::ScraperPtr albumScraper;
 598   ADDON::ScraperPtr artistScraper;
 599   if(ADDON::CAddonMgr::Get().GetDefault(ADDON::ADDON_SCRAPER_ALBUMS, addon))
 600     albumScraper = boost::dynamic_pointer_cast<ADDON::CScraper>(addon);
 601
 602   if(ADDON::CAddonMgr::Get().GetDefault(ADDON::ADDON_SCRAPER_ARTISTS, addon))
 603     artistScraper = boost::dynamic_pointer_cast<ADDON::CScraper>(addon);
 604
 605   // Add each album
 606   for (VECALBUMS::iterator album = albums.begin(); album != albums.end(); ++album)
 607   {
 608     if (m_bStop)
 609       break;
 610
 611     album->strPath = strDirectory;
 612     m_musicDatabase.BeginTransaction();
 613
 614     // Check if the album has already been downloaded or failed
 615     map<CAlbum, CAlbum>::iterator cachedAlbum = m_albumCache.find(*album);
 616     if (cachedAlbum == m_albumCache.end())
 617     {
 618       // No - download the information
 619       CMusicAlbumInfo albumInfo;
 620       INFO_RET albumDownloadStatus = INFO_NOT_FOUND;
 621       if ((m_flags & SCAN_ONLINE) && albumScraper)
 622         albumDownloadStatus = DownloadAlbumInfo(*album, albumScraper, albumInfo);
 623
 624       if (albumDownloadStatus == INFO_ADDED || albumDownloadStatus == INFO_HAVE_ALREADY)
 625       {
 626         CAlbum &downloadedAlbum = albumInfo.GetAlbum();
 627         downloadedAlbum.idAlbum = m_musicDatabase.AddAlbum(downloadedAlbum.strAlbum,
 628                                                            downloadedAlbum.strMusicBrainzAlbumID,
 629                                                            downloadedAlbum.GetArtistString(),
 630                                                            downloadedAlbum.GetGenreString(),
 631                                                            downloadedAlbum.iYear,
 632                                                            downloadedAlbum.bCompilation);
 633         m_musicDatabase.SetAlbumInfo(downloadedAlbum.idAlbum,
 634                                      downloadedAlbum,
 635                                      downloadedAlbum.songs);
 636         m_musicDatabase.SetArtForItem(downloadedAlbum.idAlbum,
 637                                       "album", album->art);
 638         GetAlbumArtwork(downloadedAlbum.idAlbum, downloadedAlbum);
 639         m_albumCache.insert(make_pair(*album, albumInfo.GetAlbum()));
 640       }
 641       else if (albumDownloadStatus == INFO_CANCELLED)
 642         break;
 643       else // Cache the lookup failure so we don't retry
 644       {
 645         album->idAlbum = m_musicDatabase.AddAlbum(album->strAlbum,
 646                                                   album->strMusicBrainzAlbumID,
 647                                                   album->GetArtistString(),
 648                                                   album->GetGenreString(),
 649                                                   album->iYear,
 650                                                   album->bCompilation);
 651         m_albumCache.insert(make_pair(*album, *album));
 652       }
 653
 654       // Update the cache pointer with our newly created info
 655       cachedAlbum = m_albumCache.find(*album);
 656     }
 657
 658     if (m_bStop)
 659       break;
 660
 661     // Add the album artists
 662     for (VECARTISTCREDITS::iterator artistCredit = cachedAlbum->second.artistCredits.begin(); artistCredit != cachedAlbum->second.artistCredits.end(); ++artistCredit)
 663     {
 664       if (m_bStop)
 665         break;
 666
 667       // Check if the artist has already been downloaded or failed
 668       map<CArtistCredit, CArtist>::iterator cachedArtist = m_artistCache.find(*artistCredit);
 669       if (cachedArtist == m_artistCache.end())
 670       {
 671         CArtist artistTmp;
 672         artistTmp.strArtist = artistCredit->GetArtist();
 673         artistTmp.strMusicBrainzArtistID = artistCredit->GetMusicBrainzArtistID();
 674         URIUtils::GetParentPath(album->strPath, artistTmp.strPath);
 675
 676         // No - download the information
 677         CMusicArtistInfo artistInfo;
 678         INFO_RET artistDownloadStatus = INFO_NOT_FOUND;
 679         if ((m_flags & SCAN_ONLINE) && artistScraper)
 680           artistDownloadStatus = DownloadArtistInfo(artistTmp, artistScraper, artistInfo);
 681
 682         if (artistDownloadStatus == INFO_ADDED || artistDownloadStatus == INFO_HAVE_ALREADY)
 683         {
 684           CArtist &downloadedArtist = artistInfo.GetArtist();
 685           downloadedArtist.idArtist = m_musicDatabase.AddArtist(downloadedArtist.strArtist,
 686                                                                 downloadedArtist.strMusicBrainzArtistID);
 687           m_musicDatabase.SetArtistInfo(downloadedArtist.idArtist,
 688                                         downloadedArtist);
 689
 690           URIUtils::GetParentPath(album->strPath, downloadedArtist.strPath);
 691           map<string, string> artwork = GetArtistArtwork(downloadedArtist);
 692           // check thumb stuff
 693           m_musicDatabase.SetArtForItem(downloadedArtist.idArtist, "artist", artwork);
 694           m_artistCache.insert(make_pair(*artistCredit, downloadedArtist));
 695         }
 696         else if (artistDownloadStatus == INFO_CANCELLED)
 697           break;
 698         else
 699         {
 700           // Cache the lookup failure so we don't retry
 701           artistTmp.idArtist = m_musicDatabase.AddArtist(artistCredit->GetArtist(), artistCredit->GetMusicBrainzArtistID());
 702           m_artistCache.insert(make_pair(*artistCredit, artistTmp));
 703         }
 704
 705         // Update the cache pointer with our newly created info
 706         cachedArtist = m_artistCache.find(*artistCredit);
 707       }
 708
 709       m_musicDatabase.AddAlbumArtist(cachedArtist->second.idArtist,
 710                                      cachedAlbum->second.idAlbum,
 711                                      artistCredit->GetJoinPhrase(),
 712                                      artistCredit == album->artistCredits.begin() ? false : true,
 713                                      std::distance(cachedAlbum->second.artistCredits.begin(), artistCredit));
 714     }
 715
 716     if (m_bStop)
 717       break;
 718
 719     for (VECSONGS::iterator song = album->songs.begin(); song != album->songs.end(); ++song)
 720     {
 721       song->idAlbum = cachedAlbum->second.idAlbum;
 722       song->idSong = m_musicDatabase.AddSong(cachedAlbum->second.idAlbum,
 723                                              song->strTitle, song->strMusicBrainzTrackID,
 724                                              song->strFileName, song->strComment,
 725                                              song->strThumb,
 726                                              song->artist, song->genre,
 727                                              song->iTrack, song->iDuration, song->iYear,
 728                                              song->iTimesPlayed, song->iStartOffset,
 729                                              song->iEndOffset,
 730                                              song->lastPlayed,
 731                                              song->rating,
 732                                              song->iKaraokeNumber);
 733       for (VECARTISTCREDITS::iterator artistCredit = song->artistCredits.begin(); artistCredit != song->artistCredits.end(); ++artistCredit)
 734       {
 735         if (m_bStop)
 736           break;
 737
 738         // Check if the artist has already been downloaded or failed
 739         map<CArtistCredit, CArtist>::iterator cachedArtist = m_artistCache.find(*artistCredit);
 740         if (cachedArtist == m_artistCache.end())
 741         {
 742           CArtist artistTmp;
 743           artistTmp.strArtist = artistCredit->GetArtist();
 744           artistTmp.strMusicBrainzArtistID = artistCredit->GetMusicBrainzArtistID();
 745           URIUtils::GetParentPath(album->strPath, artistTmp.strPath); // FIXME
 746
 747           // No - download the information
 748           CMusicArtistInfo artistInfo;
 749           INFO_RET artistDownloadStatus = INFO_NOT_FOUND;
 750           if ((m_flags & SCAN_ONLINE) && artistScraper)
 751             artistDownloadStatus = DownloadArtistInfo(artistTmp, artistScraper, artistInfo);
 752
 753           if (artistDownloadStatus == INFO_ADDED || artistDownloadStatus == INFO_HAVE_ALREADY)
 754           {
 755             CArtist &downloadedArtist = artistInfo.GetArtist();
 756             downloadedArtist.idArtist = m_musicDatabase.AddArtist(downloadedArtist.strArtist,
 757                                                                   downloadedArtist.strMusicBrainzArtistID);
 758             m_musicDatabase.SetArtistInfo(downloadedArtist.idArtist,
 759                                           downloadedArtist);
 760             // check thumb stuff
 761             URIUtils::GetParentPath(album->strPath, downloadedArtist.strPath);
 762             map<string, string> artwork = GetArtistArtwork(downloadedArtist);
 763             m_musicDatabase.SetArtForItem(downloadedArtist.idArtist, "artist", artwork);
 764             m_artistCache.insert(make_pair(*artistCredit, downloadedArtist));
 765           }
 766           else if (artistDownloadStatus == INFO_CANCELLED)
 767             break;
 768           else
 769           {
 770             // Cache the lookup failure so we don't retry
 771             artistTmp.idArtist = m_musicDatabase.AddArtist(artistCredit->GetArtist(), artistCredit->GetMusicBrainzArtistID());
 772             m_artistCache.insert(make_pair(*artistCredit, artistTmp));
 773           }
 774
 775           // Update the cache pointer with our newly created info
 776           cachedArtist = m_artistCache.find(*artistCredit);
 777         }
 778
 779         m_musicDatabase.AddSongArtist(cachedArtist->second.idArtist,
 780                                       song->idSong,
 781                                       g_advancedSettings.m_musicItemSeparator, // we don't have song artist breakdowns from scrapers, yet
 782                                       artistCredit == song->artistCredits.begin() ? false : true,
 783                                       std::distance(song->artistCredits.begin(), artistCredit));
 784       }
 785     }
 786
 787     if (m_bStop)
 788       break;
 789
 790     // Commit the album to the DB
 791     m_musicDatabase.CommitTransaction();
 792     numAdded += album->songs.size();
 793   }
 794
 795   if (m_bStop)
 796     m_musicDatabase.RollbackTransaction();
 797
 798   if (m_handle)
 799     m_handle->SetTitle(g_localizeStrings.Get(505));
 800
 801   return numAdded;
 802 }
 803
 804 static bool SortSongsByTrack(const CSong& song, const CSong& song2)
 805 {
 806   return song.iTrack < song2.iTrack;
 807 }
 808
 809 void CMusicInfoScanner::FixupAlbums(VECALBUMS &albums)
 810 {
 811   /*
 812    Step 2: Split into unique albums based on album name and album artist
 813    In the case where the album artist is unknown, we use the primary artist
 814    (i.e. first artist from each song).
 815    */
 816   for (VECALBUMS::iterator album = albums.begin(); album != albums.end(); ++album)
 817   {
 818     /*
 819      * If we have a valid MusicBrainz tag for the album, assume everything
 820      * is okay with our tags, as Picard should set everything up correctly.
 821      */
 822     if (!album->strMusicBrainzAlbumID.IsEmpty())
 823       continue;
 824
 825     VECSONGS &songs = album->songs;
 826     // sort the songs by tracknumber to identify duplicate track numbers
 827     sort(songs.begin(), songs.end(), SortSongsByTrack);
 828
 829     // map the songs to their primary artists
 830     bool tracksOverlap = false;
 831     bool hasAlbumArtist = false;
 832     bool isCompilation = true;
 833
 834     map<string, vector<CSong *> > artists;
 835     for (VECSONGS::iterator song = songs.begin(); song != songs.end(); ++song)
 836     {
 837       // test for song overlap
 838       if (song != songs.begin() && song->iTrack == (song - 1)->iTrack)
 839         tracksOverlap = true;
 840
 841       if (!song->bCompilation)
 842         isCompilation = false;
 843
 844       // get primary artist
 845       string primary;
 846       if (!song->albumArtist.empty())
 847       {
 848         primary = song->albumArtist[0];
 849         hasAlbumArtist = true;
 850       }
 851       else if (!song->artist.empty())
 852         primary = song->artist[0];
 853
 854       // add to the artist map
 855       artists[primary].push_back(&(*song));
 856     }
 857
 858     /*
 859      We have a compilation if
 860      1. album name is non-empty AND
 861      2a. no tracks overlap OR
 862      2b. all tracks are marked as part of compilation AND
 863      3a. a unique primary artist is specified as "various" or "various artists" OR
 864      3b. we have at least two primary artists and no album artist specified.
 865      */
 866     bool compilation = !album->strAlbum.empty() && (isCompilation || !tracksOverlap); // 1+2b+2a
 867     if (artists.size() == 1)
 868     {
 869       string artist = artists.begin()->first; StringUtils::ToLower(artist);
 870       if (!StringUtils::EqualsNoCase(artist, "various") &&
 871           !StringUtils::EqualsNoCase(artist, "various artists")) // 3a
 872         compilation = false;
 873     }
 874     else if (hasAlbumArtist) // 3b
 875       compilation = false;
 876
 877     if (compilation)
 878     {
 879       CLog::Log(LOGDEBUG, "Album '%s' is a compilation as there's no overlapping tracks and %s", album->strAlbum.c_str(), hasAlbumArtist ? "the album artist is 'Various'" : "there is more than one unique artist");
 880       artists.clear();
 881       std::string various = g_localizeStrings.Get(340); // Various Artists
 882       vector<string> va; va.push_back(various);
 883       for (VECSONGS::iterator song = songs.begin(); song != songs.end(); ++song)
 884       {
 885         song->albumArtist = va;
 886         artists[various].push_back(&(*song));
 887       }
 888     }
 889
 890     /*
 891      Step 3: Find the common albumartist for each song and assign
 892      albumartist to those tracks that don't have it set.
 893      */
 894     for (map<string, vector<CSong *> >::iterator j = artists.begin(); j != artists.end(); ++j)
 895     {
 896       // find the common artist for these songs
 897       vector<CSong *> &artistSongs = j->second;
 898       vector<string> common = artistSongs.front()->albumArtist.empty() ? artistSongs.front()->artist : artistSongs.front()->albumArtist;
 899       for (vector<CSong *>::iterator k = artistSongs.begin() + 1; k != artistSongs.end(); ++k)
 900       {
 901         unsigned int match = 0;
 902         vector<string> &compare = (*k)->albumArtist.empty() ? (*k)->artist : (*k)->albumArtist;
 903         for (; match < common.size() && match < compare.size(); match++)
 904         {
 905           if (compare[match] != common[match])
 906             break;
 907         }
 908         common.erase(common.begin() + match, common.end());
 909       }
 910
 911       /*
 912        Step 4: Assign the album artist for each song that doesn't have it set
 913        and add to the album vector
 914        */
 915       album->artistCredits.clear();
 916       for (vector<string>::iterator it = common.begin(); it != common.end(); ++it)
 917       {
 918         CStdString strJoinPhrase = (it == --common.end() ? "" : g_advancedSettings.m_musicItemSeparator);
 919         CArtistCredit artistCredit(*it, strJoinPhrase);
 920         album->artistCredits.push_back(artistCredit);
 921       }
 922       album->bCompilation = compilation;
 923       for (vector<CSong *>::iterator k = artistSongs.begin(); k != artistSongs.end(); ++k)
 924       {
 925         if ((*k)->albumArtist.empty())
 926           (*k)->albumArtist = common;
 927         // TODO: in future we may wish to union up the genres, for now we assume they're the same
 928         if (album->genre.empty())
 929           album->genre = (*k)->genre;
 930         //       in addition, we may want to use year as discriminating for albums
 931         if (album->iYear == 0)
 932           album->iYear = (*k)->iYear;
 933       }
 934     }
 935   }
 936 }
 937
 938 void CMusicInfoScanner::FindArtForAlbums(VECALBUMS &albums, const CStdString &path)
 939 {
 940   /*
 941    If there's a single album in the folder, then art can be taken from
 942    the folder art.
 943    */
 944   std::string albumArt;
 945   if (albums.size() == 1)
 946   {
 947     CFileItem album(path, true);
 948     albumArt = album.GetUserMusicThumb(true);
 949     if (!albumArt.empty())
 950       albums[0].art["thumb"] = albumArt;
 951   }
 952   for (VECALBUMS::iterator i = albums.begin(); i != albums.end(); ++i)
 953   {
 954     CAlbum &album = *i;
 955
 956     if (albums.size() != 1)
 957       albumArt = "";
 958
 959     /*
 960      Find art that is common across these items
 961      If we find a single art image we treat it as the album art
 962      and discard song art else we use first as album art and
 963      keep everything as song art.
 964      */
 965     bool singleArt = true;
 966     CSong *art = NULL;
 967     for (VECSONGS::iterator k = album.songs.begin(); k != album.songs.end(); ++k)
 968     {
 969       CSong &song = *k;
 970       if (song.HasArt())
 971       {
 972         if (art && !art->ArtMatches(song))
 973         {
 974           singleArt = false;
 975           break;
 976         }
 977         if (!art)
 978           art = &song;
 979       }
 980     }
 981
 982     /*
 983       assign the first art found to the album - better than no art at all
 984     */
 985
 986     if (art && albumArt.empty())
 987     {
 988       if (!art->strThumb.empty())
 989         albumArt = art->strThumb;
 990       else
 991         albumArt = CTextureCache::GetWrappedImageURL(art->strFileName, "music");
 992     }
 993
 994     if (!albumArt.empty())
 995       album.art["thumb"] = albumArt;
 996
 997     if (singleArt)
 998     { //if singleArt then we can clear the artwork for all songs
 999       for (VECSONGS::iterator k = album.songs.begin(); k != album.songs.end(); ++k)
1000         k->strThumb.clear();
1001     }
1002     else
1003     { // more than one piece of art was found for these songs, so cache per song
1004       for (VECSONGS::iterator k = album.songs.begin(); k != album.songs.end(); ++k)
1005       {
1006         if (k->strThumb.empty() && !k->embeddedArt.empty())
1007           k->strThumb = CTextureCache::GetWrappedImageURL(k->strFileName, "music");
1008       }
1009     }
1010   }
1011   if (albums.size() == 1 && !albumArt.empty())
1012   {
1013     // assign to folder thumb as well
1014     CFileItem albumItem(path, true);
1015     CMusicThumbLoader::SetCachedImage(albumItem, "thumb", albumArt);
1016   }
1017 }
1018
1019 int CMusicInfoScanner::GetPathHash(const CFileItemList &items, CStdString &hash)
1020 {
1021   // Create a hash based on the filenames, filesize and filedate.  Also count the number of files
1022   if (0 == items.Size()) return 0;
1023   XBMC::XBMC_MD5 md5state;
1024   int count = 0;
1025   for (int i = 0; i < items.Size(); ++i)
1026   {
1027     const CFileItemPtr pItem = items[i];
1028     md5state.append(pItem->GetPath());
1029     md5state.append((unsigned char *)&pItem->m_dwSize, sizeof(pItem->m_dwSize));
1030     FILETIME time = pItem->m_dateTime;
1031     md5state.append((unsigned char *)&time, sizeof(FILETIME));
1032     if (pItem->IsAudio() && !pItem->IsPlayList() && !pItem->IsNFO())
1033       count++;
1034   }
1035   md5state.getDigest(hash);
1036   return count;
1037 }
1038
1039 INFO_RET CMusicInfoScanner::UpdateDatabaseAlbumInfo(const CStdString& strPath, CMusicAlbumInfo& albumInfo, bool bAllowSelection, CGUIDialogProgress* pDialog /* = NULL */)
1040 {
1041   m_musicDatabase.Open();
1042   CQueryParams params;
1043   CDirectoryNode::GetDatabaseInfo(strPath, params);
1044
1045   if (params.GetAlbumId() == -1)
1046     return INFO_ERROR;
1047
1048   CAlbum album;
1049   m_musicDatabase.GetAlbumInfo(params.GetAlbumId(), album, &album.songs);
1050
1051   // find album info
1052   ADDON::ScraperPtr scraper;
1053   bool result = m_musicDatabase.GetScraperForPath(strPath, scraper, ADDON::ADDON_SCRAPER_ALBUMS);
1054
1055   m_musicDatabase.Close();
1056
1057   if (!result || !scraper)
1058     return INFO_ERROR;
1059
1060 loop:
1061   CLog::Log(LOGDEBUG, "%s downloading info for: %s", __FUNCTION__, album.strAlbum.c_str());
1062   INFO_RET albumDownloadStatus = DownloadAlbumInfo(album, scraper, albumInfo, pDialog);
1063   if (albumDownloadStatus == INFO_NOT_FOUND)
1064   {
1065     if (pDialog && bAllowSelection)
1066     {
1067       if (!CGUIKeyboardFactory::ShowAndGetInput(album.strAlbum, g_localizeStrings.Get(16011), false))
1068         return INFO_CANCELLED;
1069
1070       CStdString strTempArtist(StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
1071       if (!CGUIKeyboardFactory::ShowAndGetInput(strTempArtist, g_localizeStrings.Get(16025), false))
1072         return INFO_CANCELLED;
1073
1074       album.artist = StringUtils::Split(strTempArtist, g_advancedSettings.m_musicItemSeparator);
1075       goto loop;
1076     }
1077   }
1078   else if (albumDownloadStatus == INFO_ADDED)
1079   {
1080     m_musicDatabase.Open();
1081     m_musicDatabase.SetAlbumInfo(params.GetAlbumId(), albumInfo.GetAlbum(), albumInfo.GetAlbum().songs);
1082     GetAlbumArtwork(params.GetAlbumId(), albumInfo.GetAlbum());
1083     albumInfo.SetLoaded(true);
1084     m_musicDatabase.Close();
1085   }
1086   return albumDownloadStatus;
1087 }
1088
1089 INFO_RET CMusicInfoScanner::UpdateDatabaseArtistInfo(const CStdString& strPath, CMusicArtistInfo& artistInfo, bool bAllowSelection, CGUIDialogProgress* pDialog /* = NULL */)
1090 {
1091   m_musicDatabase.Open();
1092   CQueryParams params;
1093   CDirectoryNode::GetDatabaseInfo(strPath, params);
1094
1095   if (params.GetArtistId() == -1)
1096     return INFO_ERROR;
1097
1098   CArtist artist;
1099   m_musicDatabase.GetArtistInfo(params.GetArtistId(), artist);
1100
1101   // find album info
1102   ADDON::ScraperPtr scraper;
1103   if (!m_musicDatabase.GetScraperForPath(strPath, scraper, ADDON::ADDON_SCRAPER_ARTISTS) || !scraper)
1104     return INFO_ERROR;
1105   m_musicDatabase.Close();
1106
1107 loop:
1108   CLog::Log(LOGDEBUG, "%s downloading info for: %s", __FUNCTION__, artist.strArtist.c_str());
1109   INFO_RET artistDownloadStatus = DownloadArtistInfo(artist, scraper, artistInfo, pDialog);
1110   if (artistDownloadStatus == INFO_NOT_FOUND)
1111   {
1112     if (pDialog && bAllowSelection)
1113     {
1114       if (!CGUIKeyboardFactory::ShowAndGetInput(artist.strArtist, g_localizeStrings.Get(16025), false))
1115         return INFO_CANCELLED;
1116       goto loop;
1117     }
1118   }
1119   else if (artistDownloadStatus == INFO_ADDED)
1120   {
1121     m_musicDatabase.Open();
1122     m_musicDatabase.SetArtistInfo(params.GetArtistId(), artistInfo.GetArtist());
1123     m_musicDatabase.GetArtistPath(params.GetArtistId(), artist.strPath);
1124     map<string, string> artwork = GetArtistArtwork(artist);
1125     m_musicDatabase.SetArtForItem(params.GetArtistId(), "artist", artwork);
1126     artistInfo.SetLoaded();
1127     m_musicDatabase.Close();
1128   }
1129   return artistDownloadStatus;
1130 }
1131
1132 #define THRESHOLD .95f
1133
1134 INFO_RET CMusicInfoScanner::DownloadAlbumInfo(const CAlbum& album, ADDON::ScraperPtr& info, CMusicAlbumInfo& albumInfo, CGUIDialogProgress* pDialog)
1135 {
1136   if (m_handle)
1137   {
1138     m_handle->SetTitle(StringUtils::Format(g_localizeStrings.Get(20321), info->Name().c_str()));
1139     m_handle->SetText(StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator) + " - " + album.strAlbum);
1140   }
1141
1142   // clear our scraper cache
1143   info->ClearCache();
1144
1145   CMusicInfoScraper scraper(info);
1146   bool bMusicBrainz = false;
1147   if (!album.strMusicBrainzAlbumID.empty())
1148   {
1149     CScraperUrl musicBrainzURL;
1150     if (ResolveMusicBrainz(album.strMusicBrainzAlbumID, info, scraper, musicBrainzURL))
1151     {
1152       CMusicAlbumInfo albumNfo("nfo", musicBrainzURL);
1153       scraper.GetAlbums().clear();
1154       scraper.GetAlbums().push_back(albumNfo);
1155       bMusicBrainz = true;
1156     }
1157   }
1158
1159   // handle nfo files
1160   CStdString strNfo = URIUtils::AddFileToFolder(album.strPath, "album.nfo");
1161   CNfoFile::NFOResult result = CNfoFile::NO_NFO;
1162   CNfoFile nfoReader;
1163   if (XFILE::CFile::Exists(strNfo))
1164   {
1165     CLog::Log(LOGDEBUG,"Found matching nfo file: %s", strNfo.c_str());
1166     result = nfoReader.Create(strNfo, info, -1, album.strPath);
1167     if (result == CNfoFile::FULL_NFO)
1168     {
1169       CLog::Log(LOGDEBUG, "%s Got details from nfo", __FUNCTION__);
1170       nfoReader.GetDetails(albumInfo.GetAlbum());
1171       return INFO_ADDED;
1172     }
1173     else if (result == CNfoFile::URL_NFO || result == CNfoFile::COMBINED_NFO)
1174     {
1175       CScraperUrl scrUrl(nfoReader.ScraperUrl());
1176       CMusicAlbumInfo albumNfo("nfo",scrUrl);
1177       info = nfoReader.GetScraperInfo();
1178       CLog::Log(LOGDEBUG,"-- nfo-scraper: %s",info->Name().c_str());
1179       CLog::Log(LOGDEBUG,"-- nfo url: %s", scrUrl.m_url[0].m_url.c_str());
1180       scraper.SetScraperInfo(info);
1181       scraper.GetAlbums().clear();
1182       scraper.GetAlbums().push_back(albumNfo);
1183     }
1184     else
1185       CLog::Log(LOGERROR,"Unable to find an url in nfo file: %s", strNfo.c_str());
1186   }
1187
1188   if (!scraper.CheckValidOrFallback(CSettings::Get().GetString("musiclibrary.albumsscraper")))
1189   { // the current scraper is invalid, as is the default - bail
1190     CLog::Log(LOGERROR, "%s - current and default scrapers are invalid.  Pick another one", __FUNCTION__);
1191     return INFO_ERROR;
1192   }
1193
1194   if (!scraper.GetAlbumCount())
1195   {
1196     scraper.FindAlbumInfo(album.strAlbum, StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
1197
1198     while (!scraper.Completed())
1199     {
1200       if (m_bStop)
1201       {
1202         scraper.Cancel();
1203         return INFO_CANCELLED;
1204       }
1205       Sleep(1);
1206     }
1207   }
1208
1209   CGUIDialogSelect *pDlg = NULL;
1210   int iSelectedAlbum=0;
1211   if (result == CNfoFile::NO_NFO && !bMusicBrainz)
1212   {
1213     iSelectedAlbum = -1; // set negative so that we can detect a failure
1214     if (scraper.Succeeded() && scraper.GetAlbumCount() >= 1)
1215     {
1216       double bestRelevance = 0;
1217       double minRelevance = THRESHOLD;
1218       if (scraper.GetAlbumCount() > 1) // score the matches
1219       {
1220         //show dialog with all albums found
1221         if (pDialog)
1222         {
1223           pDlg = (CGUIDialogSelect*)g_windowManager.GetWindow(WINDOW_DIALOG_SELECT);
1224           pDlg->SetHeading(g_localizeStrings.Get(181).c_str());
1225           pDlg->Reset();
1226           pDlg->EnableButton(true, 413); // manual
1227         }
1228
1229         for (int i = 0; i < scraper.GetAlbumCount(); ++i)
1230         {
1231           CMusicAlbumInfo& info = scraper.GetAlbum(i);
1232           double relevance = info.GetRelevance();
1233           if (relevance < 0)
1234             relevance = CUtil::AlbumRelevance(info.GetAlbum().strAlbum, album.strAlbum, StringUtils::Join(info.GetAlbum().artist, g_advancedSettings.m_musicItemSeparator), StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
1235
1236           // if we're doing auto-selection (ie querying all albums at once, then allow 95->100% for perfect matches)
1237           // otherwise, perfect matches only
1238           if (relevance >= max(minRelevance, bestRelevance))
1239           { // we auto-select the best of these
1240             bestRelevance = relevance;
1241             iSelectedAlbum = i;
1242           }
1243           if (pDialog)
1244           {
1245             // set the label to [relevance]  album - artist
1246             CStdString strTemp;
1247             strTemp.Format("[%0.2f]  %s", relevance, info.GetTitle2());
1248             CFileItem item(strTemp);
1249             item.m_idepth = i; // use this to hold the index of the album in the scraper
1250             pDlg->Add(&item);
1251           }
1252           if (relevance > .99f) // we're so close, no reason to search further
1253             break;
1254         }
1255
1256         if (pDialog && bestRelevance < THRESHOLD)
1257         {
1258           pDlg->Sort(false);
1259           pDlg->DoModal();
1260
1261           // and wait till user selects one
1262           if (pDlg->GetSelectedLabel() < 0)
1263           { // none chosen
1264             if (!pDlg->IsButtonPressed())
1265               return INFO_CANCELLED;
1266
1267             // manual button pressed
1268             CStdString strNewAlbum = album.strAlbum;
1269             if (!CGUIKeyboardFactory::ShowAndGetInput(strNewAlbum, g_localizeStrings.Get(16011), false)) return INFO_CANCELLED;
1270             if (strNewAlbum == "") return INFO_CANCELLED;
1271
1272             CStdString strNewArtist = StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator);
1273             if (!CGUIKeyboardFactory::ShowAndGetInput(strNewArtist, g_localizeStrings.Get(16025), false)) return INFO_CANCELLED;
1274
1275             pDialog->SetLine(0, strNewAlbum);
1276             pDialog->SetLine(1, strNewArtist);
1277             pDialog->Progress();
1278
1279             CAlbum newAlbum = album;
1280             newAlbum.strAlbum = strNewAlbum;
1281             newAlbum.artist = StringUtils::Split(strNewArtist, g_advancedSettings.m_musicItemSeparator);
1282
1283             return DownloadAlbumInfo(newAlbum, info, albumInfo, pDialog);
1284           }
1285           iSelectedAlbum = pDlg->GetSelectedItem()->m_idepth;
1286         }
1287       }
1288       else
1289       {
1290         CMusicAlbumInfo& info = scraper.GetAlbum(0);
1291         double relevance = info.GetRelevance();
1292         if (relevance < 0)
1293           relevance = CUtil::AlbumRelevance(info.GetAlbum().strAlbum,
1294                                             album.strAlbum,
1295                                             StringUtils::Join(info.GetAlbum().artist, g_advancedSettings.m_musicItemSeparator),
1296                                             StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
1297         if (relevance < THRESHOLD)
1298           return INFO_NOT_FOUND;
1299
1300         iSelectedAlbum = 0;
1301       }
1302     }
1303
1304     if (iSelectedAlbum < 0)
1305       return INFO_NOT_FOUND;
1306
1307   }
1308
1309   scraper.LoadAlbumInfo(iSelectedAlbum);
1310   while (!scraper.Completed())
1311   {
1312     if (m_bStop)
1313     {
1314       scraper.Cancel();
1315       return INFO_CANCELLED;
1316     }
1317     Sleep(1);
1318   }
1319
1320   if (!scraper.Succeeded())
1321     return INFO_ERROR;
1322
1323   albumInfo = scraper.GetAlbum(iSelectedAlbum);
1324
1325   if (result == CNfoFile::COMBINED_NFO)
1326     nfoReader.GetDetails(albumInfo.GetAlbum(), NULL, true);
1327
1328   return INFO_ADDED;
1329 }
1330
1331 void CMusicInfoScanner::GetAlbumArtwork(long id, const CAlbum &album)
1332 {
1333   if (album.thumbURL.m_url.size())
1334   {
1335     if (m_musicDatabase.GetArtForItem(id, "album", "thumb").empty())
1336     {
1337       string thumb = CScraperUrl::GetThumbURL(album.thumbURL.GetFirstThumb());
1338       if (!thumb.empty())
1339       {
1340         CTextureCache::Get().BackgroundCacheImage(thumb);
1341         m_musicDatabase.SetArtForItem(id, "album", "thumb", thumb);
1342       }
1343     }
1344   }
1345 }
1346
1347 INFO_RET CMusicInfoScanner::DownloadArtistInfo(const CArtist& artist, ADDON::ScraperPtr& info, MUSIC_GRABBER::CMusicArtistInfo& artistInfo, CGUIDialogProgress* pDialog)
1348 {
1349   if (m_handle)
1350   {
1351     m_handle->SetTitle(StringUtils::Format(g_localizeStrings.Get(20320), info->Name().c_str()));
1352     m_handle->SetText(artist.strArtist);
1353   }
1354
1355   // clear our scraper cache
1356   info->ClearCache();
1357
1358   CMusicInfoScraper scraper(info);
1359   bool bMusicBrainz = false;
1360   if (!artist.strMusicBrainzArtistID.empty())
1361   {
1362     CScraperUrl musicBrainzURL;
1363     if (ResolveMusicBrainz(artist.strMusicBrainzArtistID, info, scraper, musicBrainzURL))
1364     {
1365       CMusicArtistInfo artistNfo("nfo", musicBrainzURL);
1366       scraper.GetArtists().clear();
1367       scraper.GetArtists().push_back(artistNfo);
1368       bMusicBrainz = true;
1369     }
1370   }
1371
1372   // handle nfo files
1373   CStdString strNfo = URIUtils::AddFileToFolder(artist.strPath, "artist.nfo");
1374   CNfoFile::NFOResult result=CNfoFile::NO_NFO;
1375   CNfoFile nfoReader;
1376   if (XFILE::CFile::Exists(strNfo))
1377   {
1378     CLog::Log(LOGDEBUG,"Found matching nfo file: %s", strNfo.c_str());
1379     result = nfoReader.Create(strNfo, info);
1380     if (result == CNfoFile::FULL_NFO)
1381     {
1382       CLog::Log(LOGDEBUG, "%s Got details from nfo", __FUNCTION__);
1383       nfoReader.GetDetails(artistInfo.GetArtist());
1384       return INFO_ADDED;
1385     }
1386     else if (result == CNfoFile::URL_NFO || result == CNfoFile::COMBINED_NFO)
1387     {
1388       CScraperUrl scrUrl(nfoReader.ScraperUrl());
1389       CMusicArtistInfo artistNfo("nfo",scrUrl);
1390       info = nfoReader.GetScraperInfo();
1391       CLog::Log(LOGDEBUG,"-- nfo-scraper: %s",info->Name().c_str());
1392       CLog::Log(LOGDEBUG,"-- nfo url: %s", scrUrl.m_url[0].m_url.c_str());
1393       scraper.SetScraperInfo(info);
1394       scraper.GetArtists().push_back(artistNfo);
1395     }
1396     else
1397       CLog::Log(LOGERROR,"Unable to find an url in nfo file: %s", strNfo.c_str());
1398   }
1399
1400   if (!scraper.GetArtistCount())
1401   {
1402     scraper.FindArtistInfo(artist.strArtist);
1403
1404     while (!scraper.Completed())
1405     {
1406       if (m_bStop)
1407       {
1408         scraper.Cancel();
1409         return INFO_CANCELLED;
1410       }
1411       Sleep(1);
1412     }
1413   }
1414
1415   int iSelectedArtist = 0;
1416   if (result == CNfoFile::NO_NFO && !bMusicBrainz)
1417   {
1418     if (scraper.GetArtistCount() >= 1)
1419     {
1420       // now load the first match
1421       if (pDialog && scraper.GetArtistCount() > 1)
1422       {
1423         // if we found more then 1 album, let user choose one
1424         CGUIDialogSelect *pDlg = (CGUIDialogSelect*)g_windowManager.GetWindow(WINDOW_DIALOG_SELECT);
1425         if (pDlg)
1426         {
1427           pDlg->SetHeading(g_localizeStrings.Get(21890));
1428           pDlg->Reset();
1429           pDlg->EnableButton(true, 413); // manual
1430
1431           for (int i = 0; i < scraper.GetArtistCount(); ++i)
1432           {
1433             // set the label to artist
1434             CFileItem item(scraper.GetArtist(i).GetArtist());
1435             CStdString strTemp=scraper.GetArtist(i).GetArtist().strArtist;
1436             if (!scraper.GetArtist(i).GetArtist().strBorn.IsEmpty())
1437               strTemp += " ("+scraper.GetArtist(i).GetArtist().strBorn+")";
1438             if (!scraper.GetArtist(i).GetArtist().genre.empty())
1439             {
1440               CStdString genres = StringUtils::Join(scraper.GetArtist(i).GetArtist().genre, g_advancedSettings.m_musicItemSeparator);
1441               if (!genres.empty())
1442                 strTemp.Format("[%s] %s", genres.c_str(), strTemp.c_str());
1443             }
1444             item.SetLabel(strTemp);
1445             item.m_idepth = i; // use this to hold the index of the album in the scraper
1446             pDlg->Add(&item);
1447           }
1448           pDlg->DoModal();
1449
1450           // and wait till user selects one
1451           if (pDlg->GetSelectedLabel() < 0)
1452           { // none chosen
1453             if (!pDlg->IsButtonPressed())
1454               return INFO_CANCELLED;
1455
1456             // manual button pressed
1457             CStdString strNewArtist = artist.strArtist;
1458             if (!CGUIKeyboardFactory::ShowAndGetInput(strNewArtist, g_localizeStrings.Get(16025), false)) return INFO_CANCELLED;
1459
1460             if (pDialog)
1461             {
1462               pDialog->SetLine(0, strNewArtist);
1463               pDialog->Progress();
1464             }
1465
1466             CArtist newArtist;
1467             newArtist.strArtist = strNewArtist;
1468             return DownloadArtistInfo(newArtist, info, artistInfo, pDialog);
1469           }
1470           iSelectedArtist = pDlg->GetSelectedItem()->m_idepth;
1471         }
1472       }
1473     }
1474     else
1475       return INFO_NOT_FOUND;
1476   }
1477
1478   scraper.LoadArtistInfo(iSelectedArtist, artist.strArtist);
1479   while (!scraper.Completed())
1480   {
1481     if (m_bStop)
1482     {
1483       scraper.Cancel();
1484       return INFO_CANCELLED;
1485     }
1486     Sleep(1);
1487   }
1488
1489   if (!scraper.Succeeded())
1490     return INFO_ERROR;
1491
1492   artistInfo = scraper.GetArtist(iSelectedArtist);
1493
1494   if (result == CNfoFile::COMBINED_NFO)
1495     nfoReader.GetDetails(artistInfo.GetArtist(), NULL, true);
1496
1497   return INFO_ADDED;
1498 }
1499
1500 bool CMusicInfoScanner::ResolveMusicBrainz(const CStdString strMusicBrainzID, ScraperPtr &preferredScraper, CMusicInfoScraper &musicInfoScraper, CScraperUrl &musicBrainzURL)
1501 {
1502   // We have a MusicBrainz ID
1503   // Get a scraper that can resolve it to a MusicBrainz URL & force our
1504   // search directly to the specific album.
1505   bool bMusicBrainz = false;
1506   ADDON::TYPE type = ScraperTypeFromContent(preferredScraper->Content());
1507
1508   CFileItemList items;
1509   ADDON::AddonPtr addon;
1510   ADDON::ScraperPtr defaultScraper;
1511   if (ADDON::CAddonMgr::Get().GetDefault(type, addon))
1512     defaultScraper = boost::dynamic_pointer_cast<CScraper>(addon);
1513
1514   vector<ScraperPtr> vecScrapers;
1515
1516   // add selected scraper - first proirity
1517   if (preferredScraper)
1518     vecScrapers.push_back(preferredScraper);
1519
1520   // Add all scrapers except selected and default
1521   VECADDONS addons;
1522   CAddonMgr::Get().GetAddons(type, addons);
1523
1524   for (unsigned i = 0; i < addons.size(); ++i)
1525   {
1526     ScraperPtr scraper = boost::dynamic_pointer_cast<CScraper>(addons[i]);
1527
1528     // skip if scraper requires settings and there's nothing set yet
1529     if (!scraper || (scraper->RequiresSettings() && !scraper->HasUserSettings()))
1530       continue;
1531
1532     if((!preferredScraper || preferredScraper->ID() != scraper->ID()) && (!defaultScraper || defaultScraper->ID() != scraper->ID()) )
1533       vecScrapers.push_back(scraper);
1534   }
1535
1536   // add default scraper - not user selectable so it's last priority
1537   if(defaultScraper &&
1538      (!preferredScraper || preferredScraper->ID() != defaultScraper->ID()) &&
1539      (!defaultScraper->RequiresSettings() || defaultScraper->HasUserSettings()))
1540     vecScrapers.push_back(defaultScraper);
1541
1542   for (unsigned int i=0; i < vecScrapers.size(); ++i)
1543   {
1544     if (vecScrapers[i]->Type() != type)
1545       continue;
1546
1547     vecScrapers[i]->ClearCache();
1548     try
1549     {
1550       musicBrainzURL = vecScrapers[i]->ResolveIDToUrl(strMusicBrainzID);
1551     }
1552     catch (const ADDON::CScraperError &sce)
1553     {
1554       if (!sce.FAborted())
1555         continue;
1556     }
1557     if (!musicBrainzURL.m_url.empty())
1558     {
1559       Sleep(2000); // MusicBrainz rate-limits queries to 1 p.s - once we hit the rate-limiter
1560                    // they start serving up the 'you hit the rate-limiter' page fast - meaning
1561                    // we will never get below the rate-limit threshold again in a specific run.
1562                    // This helps us to avoidthe rate-limiter as far as possible.
1563       CLog::Log(LOGDEBUG,"-- nfo-scraper: %s",vecScrapers[i]->Name().c_str());
1564       CLog::Log(LOGDEBUG,"-- nfo url: %s", musicBrainzURL.m_url[0].m_url.c_str());
1565       musicInfoScraper.SetScraperInfo(vecScrapers[i]);
1566       bMusicBrainz = true;
1567       break;
1568     }
1569   }
1570
1571   return bMusicBrainz;
1572 }
1573
1574 map<string, string> CMusicInfoScanner::GetArtistArtwork(const CArtist& artist)
1575 {
1576   map<string, string> artwork;
1577
1578   // check thumb
1579   CStdString strFolder;
1580   CStdString thumb;
1581   if (!artist.strPath.IsEmpty())
1582   {
1583     strFolder = artist.strPath;
1584     for (int i = 0; i < 3 && thumb.IsEmpty(); ++i)
1585     {
1586       CFileItem item(strFolder, true);
1587       thumb = item.GetUserMusicThumb(true);
1588       strFolder = URIUtils::GetParentPath(strFolder);
1589     }
1590   }
1591   if (thumb.IsEmpty())
1592     thumb = CScraperUrl::GetThumbURL(artist.thumbURL.GetFirstThumb());
1593   if (!thumb.IsEmpty())
1594   {
1595     CTextureCache::Get().BackgroundCacheImage(thumb);
1596     artwork.insert(make_pair("thumb", thumb));
1597   }
1598
1599   // check fanart
1600   CStdString fanart;
1601   if (!artist.strPath.IsEmpty())
1602   {
1603     strFolder = artist.strPath;
1604     for (int i = 0; i < 3 && fanart.IsEmpty(); ++i)
1605     {
1606       CFileItem item(strFolder, true);
1607       fanart = item.GetLocalFanart();
1608       strFolder = URIUtils::GetParentPath(strFolder);
1609     }
1610   }
1611   if (fanart.IsEmpty())
1612     fanart = artist.fanart.GetImageURL();
1613   if (!fanart.IsEmpty())
1614   {
1615     CTextureCache::Get().BackgroundCacheImage(fanart);
1616     artwork.insert(make_pair("fanart", fanart));
1617   }
1618
1619   return artwork;
1620 }
1621
1622 // This function is the Run() function of the IRunnable
1623 // CFileCountReader and runs in a separate thread.
1624 void CMusicInfoScanner::Run()
1625 {
1626   int count = 0;
1627   for (set<std::string>::iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end() && !m_bStop; ++it)
1628   {
1629     count+=CountFilesRecursively(*it);
1630   }
1631   m_itemCount = count;
1632 }
1633
1634 // Recurse through all folders we scan and count files
1635 int CMusicInfoScanner::CountFilesRecursively(const CStdString& strPath)
1636 {
1637   // load subfolder
1638   CFileItemList items;
1639   CDirectory::GetDirectory(strPath, items, g_advancedSettings.m_musicExtensions, DIR_FLAG_NO_FILE_DIRS);
1640
1641   if (m_bStop)
1642     return 0;
1643
1644   // true for recursive counting
1645   int count = CountFiles(items, true);
1646   return count;
1647 }
1648
1649 int CMusicInfoScanner::CountFiles(const CFileItemList &items, bool recursive)
1650 {
1651   int count = 0;
1652   for (int i=0; i<items.Size(); ++i)
1653   {
1654     const CFileItemPtr pItem=items[i];
1655
1656     if (recursive && pItem->m_bIsFolder)
1657       count+=CountFilesRecursively(pItem->GetPath());
1658     else if (pItem->IsAudio() && !pItem->IsPlayList() && !pItem->IsNFO())
1659       count++;
1660   }
1661   return count;
1662 }