code.vuplus.com Git - vuplus_xbmc/blob - xbmc/music/infoscanner/MusicInfoScanner.cpp

   1 /*
   2  *      Copyright (C) 2005-2013 Team XBMC
   3  *      http://xbmc.org
   4  *
   5  *  This Program is free software; you can redistribute it and/or modify
   6  *  it under the terms of the GNU General Public License as published by
   7  *  the Free Software Foundation; either version 2, or (at your option)
   8  *  any later version.
   9  *
  10  *  This Program is distributed in the hope that it will be useful,
  11  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  *  GNU General Public License for more details.
  14  *
  15  *  You should have received a copy of the GNU General Public License
  16  *  along with XBMC; see the file COPYING.  If not, see
  17  *  <http://www.gnu.org/licenses/>.
  18  *
  19  */
  20
  21 #include "threads/SystemClock.h"
  22 #include "MusicInfoScanner.h"
  23 #include "music/tags/MusicInfoTagLoaderFactory.h"
  24 #include "MusicAlbumInfo.h"
  25 #include "MusicInfoScraper.h"
  26 #include "filesystem/MusicDatabaseDirectory.h"
  27 #include "filesystem/MusicDatabaseDirectory/DirectoryNode.h"
  28 #include "Util.h"
  29 #include "utils/md5.h"
  30 #include "GUIInfoManager.h"
  31 #include "utils/Variant.h"
  32 #include "NfoFile.h"
  33 #include "music/tags/MusicInfoTag.h"
  34 #include "guilib/GUIWindowManager.h"
  35 #include "dialogs/GUIDialogExtendedProgressBar.h"
  36 #include "dialogs/GUIDialogProgress.h"
  37 #include "dialogs/GUIDialogSelect.h"
  38 #include "guilib/GUIKeyboardFactory.h"
  39 #include "filesystem/File.h"
  40 #include "filesystem/Directory.h"
  41 #include "settings/AdvancedSettings.h"
  42 #include "settings/Settings.h"
  43 #include "FileItem.h"
  44 #include "guilib/LocalizeStrings.h"
  45 #include "utils/StringUtils.h"
  46 #include "utils/TimeUtils.h"
  47 #include "utils/log.h"
  48 #include "utils/URIUtils.h"
  49 #include "TextureCache.h"
  50 #include "music/MusicThumbLoader.h"
  51 #include "interfaces/AnnouncementManager.h"
  52 #include "GUIUserMessages.h"
  53 #include "addons/AddonManager.h"
  54 #include "addons/Scraper.h"
  55
  56 #include <algorithm>
  57
  58 using namespace std;
  59 using namespace MUSIC_INFO;
  60 using namespace XFILE;
  61 using namespace MUSICDATABASEDIRECTORY;
  62 using namespace MUSIC_GRABBER;
  63 using namespace ADDON;
  64
  65 CMusicInfoScanner::CMusicInfoScanner() : CThread("MusicInfoScanner"), m_fileCountReader(this, "MusicFileCounter")
  66 {
  67   m_bRunning = false;
  68   m_showDialog = false;
  69   m_handle = NULL;
  70   m_bCanInterrupt = false;
  71   m_currentItem=0;
  72   m_itemCount=0;
  73   m_flags = 0;
  74 }
  75
  76 CMusicInfoScanner::~CMusicInfoScanner()
  77 {
  78 }
  79
  80 void CMusicInfoScanner::Process()
  81 {
  82   ANNOUNCEMENT::CAnnouncementManager::Announce(ANNOUNCEMENT::AudioLibrary, "xbmc", "OnScanStarted");
  83   try
  84   {
  85     unsigned int tick = XbmcThreads::SystemClockMillis();
  86
  87     m_musicDatabase.Open();
  88
  89     if (m_showDialog && !CSettings::Get().GetBool("musiclibrary.backgroundupdate"))
  90     {
  91       CGUIDialogExtendedProgressBar* dialog =
  92         (CGUIDialogExtendedProgressBar*)g_windowManager.GetWindow(WINDOW_DIALOG_EXT_PROGRESS);
  93       m_handle = dialog->GetHandle(g_localizeStrings.Get(314));
  94     }
  95
  96     m_bCanInterrupt = true;
  97
  98     if (m_scanType == 0) // load info from files
  99     {
 100       CLog::Log(LOGDEBUG, "%s - Starting scan", __FUNCTION__);
 101
 102       if (m_handle)
 103         m_handle->SetTitle(g_localizeStrings.Get(505));
 104
 105       // Reset progress vars
 106       m_currentItem=0;
 107       m_itemCount=-1;
 108
 109       // Create the thread to count all files to be scanned
 110       SetPriority( GetMinPriority() );
 111       if (m_handle)
 112         m_fileCountReader.Create();
 113
 114       // Database operations should not be canceled
 115       // using Interupt() while scanning as it could
 116       // result in unexpected behaviour.
 117       m_bCanInterrupt = false;
 118       m_needsCleanup = false;
 119
 120       bool commit = false;
 121       bool cancelled = false;
 122       for (std::set<std::string>::const_iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end(); it++)
 123       {
 124         if (!CDirectory::Exists(*it) && !m_bClean)
 125         {
 126           /*
 127            * Note that this will skip scanning (if m_bClean is disabled) if the directory really
 128            * doesn't exist. Since the music scanner is fed with a list of existing paths from the DB
 129            * and cleans out all songs under that path as its first step before re-adding files, if
 130            * the entire source is offline we totally empty the music database in one go.
 131            */
 132           CLog::Log(LOGWARNING, "%s directory '%s' does not exist - skipping scan.", __FUNCTION__, it->c_str());
 133           continue;
 134         }
 135         else if (!DoScan(*it))
 136           cancelled = true;
 137         commit = !cancelled;
 138       }
 139
 140       if (commit)
 141       {
 142         g_infoManager.ResetLibraryBools();
 143
 144         if (m_needsCleanup)
 145         {
 146           if (m_handle)
 147           {
 148             m_handle->SetTitle(g_localizeStrings.Get(700));
 149             m_handle->SetText("");
 150           }
 151
 152           m_musicDatabase.CleanupOrphanedItems();
 153
 154           if (m_handle)
 155             m_handle->SetTitle(g_localizeStrings.Get(331));
 156
 157           m_musicDatabase.Compress(false);
 158         }
 159       }
 160
 161       m_fileCountReader.StopThread();
 162
 163       m_musicDatabase.EmptyCache();
 164
 165       tick = XbmcThreads::SystemClockMillis() - tick;
 166       CLog::Log(LOGNOTICE, "My Music: Scanning for music info using worker thread, operation took %s", StringUtils::SecondsToTimeString(tick / 1000).c_str());
 167     }
 168     if (m_scanType == 1) // load album info
 169     {
 170       for (std::set<std::string>::const_iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end(); ++it)
 171       {
 172         CQueryParams params;
 173         CDirectoryNode::GetDatabaseInfo(*it, params);
 174         if (m_musicDatabase.HasAlbumInfo(params.GetArtistId())) // should this be here?
 175           continue;
 176
 177         CAlbum album;
 178         m_musicDatabase.GetAlbumInfo(params.GetAlbumId(), album, &album.songs);
 179         if (m_handle)
 180         {
 181           float percentage = (float) std::distance(it, m_pathsToScan.end()) / m_pathsToScan.size();
 182           m_handle->SetText(StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator) + " - " + album.strAlbum);
 183           m_handle->SetPercentage(percentage);
 184         }
 185
 186         CMusicAlbumInfo albumInfo;
 187         UpdateDatabaseAlbumInfo(*it, albumInfo, false);
 188
 189         if (m_bStop)
 190           break;
 191       }
 192     }
 193     if (m_scanType == 2) // load artist info
 194     {
 195       for (std::set<std::string>::const_iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end(); ++it)
 196       {
 197         CQueryParams params;
 198         CDirectoryNode::GetDatabaseInfo(*it, params);
 199         if (m_musicDatabase.HasArtistInfo(params.GetArtistId())) // should this be here?
 200             continue;
 201
 202         CArtist artist;
 203         m_musicDatabase.GetArtistInfo(params.GetArtistId(), artist);
 204         m_musicDatabase.GetArtistPath(params.GetArtistId(), artist.strPath);
 205
 206         if (m_handle)
 207         {
 208           float percentage = (float) (std::distance(m_pathsToScan.begin(), it) / m_pathsToScan.size()) * 100;
 209           m_handle->SetText(artist.strArtist);
 210           m_handle->SetPercentage(percentage);
 211         }
 212
 213         CMusicArtistInfo artistInfo;
 214         UpdateDatabaseArtistInfo(*it, artistInfo, false);
 215
 216         if (m_bStop)
 217           break;
 218       }
 219     }
 220
 221   }
 222   catch (...)
 223   {
 224     CLog::Log(LOGERROR, "MusicInfoScanner: Exception while scanning.");
 225   }
 226   m_musicDatabase.Close();
 227   CLog::Log(LOGDEBUG, "%s - Finished scan", __FUNCTION__);
 228
 229   m_bRunning = false;
 230   ANNOUNCEMENT::CAnnouncementManager::Announce(ANNOUNCEMENT::AudioLibrary, "xbmc", "OnScanFinished");
 231
 232   // we need to clear the musicdb cache and update any active lists
 233   CUtil::DeleteMusicDatabaseDirectoryCache();
 234   CGUIMessage msg(GUI_MSG_SCAN_FINISHED, 0, 0, 0);
 235   g_windowManager.SendThreadMessage(msg);
 236
 237   if (m_handle)
 238     m_handle->MarkFinished();
 239   m_handle = NULL;
 240 }
 241
 242 void CMusicInfoScanner::Start(const CStdString& strDirectory, int flags)
 243 {
 244   m_fileCountReader.StopThread();
 245   StopThread();
 246   m_pathsToScan.clear();
 247   m_flags = flags;
 248
 249   if (strDirectory.IsEmpty())
 250   { // scan all paths in the database.  We do this by scanning all paths in the db, and crossing them off the list as
 251     // we go.
 252     m_musicDatabase.Open();
 253     m_musicDatabase.GetPaths(m_pathsToScan);
 254     m_musicDatabase.Close();
 255   }
 256   else
 257     m_pathsToScan.insert(strDirectory);
 258   m_bClean = g_advancedSettings.m_bMusicLibraryCleanOnUpdate;
 259
 260   m_scanType = 0;
 261   Create();
 262   m_bRunning = true;
 263 }
 264
 265 void CMusicInfoScanner::FetchAlbumInfo(const CStdString& strDirectory,
 266                                        bool refresh)
 267 {
 268   m_fileCountReader.StopThread();
 269   StopThread();
 270   m_pathsToScan.clear();
 271
 272   CFileItemList items;
 273   if (strDirectory.IsEmpty())
 274   {
 275     m_musicDatabase.Open();
 276     m_musicDatabase.GetAlbumsNav("musicdb://albums/", items);
 277     m_musicDatabase.Close();
 278   }
 279   else
 280   {
 281     if (URIUtils::HasSlashAtEnd(strDirectory)) // directory
 282       CDirectory::GetDirectory(strDirectory,items);
 283     else
 284     {
 285       CFileItemPtr item(new CFileItem(strDirectory,false));
 286       items.Add(item);
 287     }
 288   }
 289
 290   m_musicDatabase.Open();
 291   for (int i=0;i<items.Size();++i)
 292   {
 293     if (CMusicDatabaseDirectory::IsAllItem(items[i]->GetPath()) || items[i]->IsParentFolder())
 294       continue;
 295
 296     m_pathsToScan.insert(items[i]->GetPath());
 297     if (refresh)
 298     {
 299       m_musicDatabase.DeleteAlbumInfo(items[i]->GetMusicInfoTag()->GetDatabaseId());
 300     }
 301   }
 302   m_musicDatabase.Close();
 303
 304   m_scanType = 1;
 305   Create();
 306   m_bRunning = true;
 307 }
 308
 309 void CMusicInfoScanner::FetchArtistInfo(const CStdString& strDirectory,
 310                                         bool refresh)
 311 {
 312   m_fileCountReader.StopThread();
 313   StopThread();
 314   m_pathsToScan.clear();
 315   CFileItemList items;
 316
 317   if (strDirectory.IsEmpty())
 318   {
 319     m_musicDatabase.Open();
 320     m_musicDatabase.GetArtistsNav("musicdb://artists/", items, false, -1);
 321     m_musicDatabase.Close();
 322   }
 323   else
 324   {
 325     if (URIUtils::HasSlashAtEnd(strDirectory)) // directory
 326       CDirectory::GetDirectory(strDirectory,items);
 327     else
 328     {
 329       CFileItemPtr newItem(new CFileItem(strDirectory,false));
 330       items.Add(newItem);
 331     }
 332   }
 333
 334   m_musicDatabase.Open();
 335   for (int i=0;i<items.Size();++i)
 336   {
 337     if (CMusicDatabaseDirectory::IsAllItem(items[i]->GetPath()) || items[i]->IsParentFolder())
 338       continue;
 339
 340     m_pathsToScan.insert(items[i]->GetPath());
 341     if (refresh)
 342     {
 343       m_musicDatabase.DeleteArtistInfo(items[i]->GetMusicInfoTag()->GetDatabaseId());
 344     }
 345   }
 346   m_musicDatabase.Close();
 347
 348   m_scanType = 2;
 349   Create();
 350   m_bRunning = true;
 351 }
 352
 353 bool CMusicInfoScanner::IsScanning()
 354 {
 355   return m_bRunning;
 356 }
 357
 358 void CMusicInfoScanner::Stop()
 359 {
 360   if (m_bCanInterrupt)
 361     m_musicDatabase.Interupt();
 362
 363   StopThread(false);
 364 }
 365
 366 static void OnDirectoryScanned(const CStdString& strDirectory)
 367 {
 368   CGUIMessage msg(GUI_MSG_DIRECTORY_SCANNED, 0, 0, 0);
 369   msg.SetStringParam(strDirectory);
 370   g_windowManager.SendThreadMessage(msg);
 371 }
 372
 373 static CStdString Prettify(const CStdString& strDirectory)
 374 {
 375   CURL url(strDirectory);
 376   CStdString strStrippedPath = url.GetWithoutUserDetails();
 377   CURL::Decode(strStrippedPath);
 378
 379   return strStrippedPath;
 380 }
 381
 382 bool CMusicInfoScanner::DoScan(const CStdString& strDirectory)
 383 {
 384   if (m_handle)
 385     m_handle->SetText(Prettify(strDirectory));
 386
 387   // Discard all excluded files defined by m_musicExcludeRegExps
 388   CStdStringArray regexps = g_advancedSettings.m_audioExcludeFromScanRegExps;
 389   if (CUtil::ExcludeFileOrFolder(strDirectory, regexps))
 390     return true;
 391
 392   // load subfolder
 393   CFileItemList items;
 394   CDirectory::GetDirectory(strDirectory, items, g_advancedSettings.m_musicExtensions + "|.jpg|.tbn|.lrc|.cdg");
 395
 396   // sort and get the path hash.  Note that we don't filter .cue sheet items here as we want
 397   // to detect changes in the .cue sheet as well.  The .cue sheet items only need filtering
 398   // if we have a changed hash.
 399   items.Sort(SortByLabel, SortOrderAscending);
 400   CStdString hash;
 401   GetPathHash(items, hash);
 402
 403   // check whether we need to rescan or not
 404   CStdString dbHash;
 405   if ((m_flags & SCAN_RESCAN) || !m_musicDatabase.GetPathHash(strDirectory, dbHash) || dbHash != hash)
 406   { // path has changed - rescan
 407     if (dbHash.IsEmpty())
 408       CLog::Log(LOGDEBUG, "%s Scanning dir '%s' as not in the database", __FUNCTION__, strDirectory.c_str());
 409     else
 410       CLog::Log(LOGDEBUG, "%s Rescanning dir '%s' due to change", __FUNCTION__, strDirectory.c_str());
 411
 412     // filter items in the sub dir (for .cue sheet support)
 413     items.FilterCueItems();
 414     items.Sort(SortByLabel, SortOrderAscending);
 415
 416     // and then scan in the new information
 417     if (RetrieveMusicInfo(strDirectory, items) > 0)
 418     {
 419       if (m_handle)
 420         OnDirectoryScanned(strDirectory);
 421     }
 422
 423     // save information about this folder
 424     m_musicDatabase.SetPathHash(strDirectory, hash);
 425   }
 426   else
 427   { // path is the same - no need to rescan
 428     CLog::Log(LOGDEBUG, "%s Skipping dir '%s' due to no change", __FUNCTION__, strDirectory.c_str());
 429     m_currentItem += CountFiles(items, false);  // false for non-recursive
 430
 431     // updated the dialog with our progress
 432     if (m_handle)
 433     {
 434       if (m_itemCount>0)
 435         m_handle->SetPercentage(m_currentItem/(float)m_itemCount*100);
 436       OnDirectoryScanned(strDirectory);
 437     }
 438   }
 439
 440   // now scan the subfolders
 441   for (int i = 0; i < items.Size(); ++i)
 442   {
 443     CFileItemPtr pItem = items[i];
 444
 445     if (m_bStop)
 446       break;
 447     // if we have a directory item (non-playlist) we then recurse into that folder
 448     if (pItem->m_bIsFolder && !pItem->IsParentFolder() && !pItem->IsPlayList())
 449     {
 450       CStdString strPath=pItem->GetPath();
 451       if (!DoScan(strPath))
 452       {
 453         m_bStop = true;
 454       }
 455     }
 456   }
 457
 458   return !m_bStop;
 459 }
 460
 461 INFO_RET CMusicInfoScanner::ScanTags(const CFileItemList& items, CFileItemList& scannedItems)
 462 {
 463   CStdStringArray regexps = g_advancedSettings.m_audioExcludeFromScanRegExps;
 464
 465   for (int i = 0; i < items.Size(); ++i)
 466   {
 467     if (m_bStop)
 468       return INFO_CANCELLED;
 469
 470     CFileItemPtr pItem = items[i];
 471
 472     if (CUtil::ExcludeFileOrFolder(pItem->GetPath(), regexps))
 473       continue;
 474
 475     if (pItem->m_bIsFolder || pItem->IsPlayList() || pItem->IsPicture() || pItem->IsLyrics())
 476       continue;
 477
 478     m_currentItem++;
 479
 480     CMusicInfoTag& tag = *pItem->GetMusicInfoTag();
 481     if (!tag.Loaded())
 482     {
 483       auto_ptr<IMusicInfoTagLoader> pLoader (CMusicInfoTagLoaderFactory::CreateLoader(pItem->GetPath()));
 484       if (NULL != pLoader.get())
 485         pLoader->Load(pItem->GetPath(), tag);
 486     }
 487
 488     if (m_handle && m_itemCount>0)
 489       m_handle->SetPercentage(m_currentItem/(float)m_itemCount*100);
 490
 491     if (!tag.Loaded())
 492     {
 493       CLog::Log(LOGDEBUG, "%s - No tag found for: %s", __FUNCTION__, pItem->GetPath().c_str());
 494       continue;
 495     }
 496     scannedItems.Add(pItem);
 497   }
 498   return INFO_ADDED;
 499 }
 500
 501 static bool SortSongsByTrack(const CSong& song, const CSong& song2)
 502 {
 503   return song.iTrack < song2.iTrack;
 504 }
 505
 506 void CMusicInfoScanner::FileItemsToAlbums(CFileItemList& items, VECALBUMS& albums, MAPSONGS* songsMap /* = NULL */)
 507 {
 508   /*
 509    * Step 1: Convert the FileItems into Songs.
 510    * If they're MB tagged, create albums directly from the FileItems.
 511    * If they're non-MB tagged, index them by album name ready for step 2.
 512    */
 513   map<string, VECSONGS> songsByAlbumNames;
 514   for (int i = 0; i < items.Size(); ++i)
 515   {
 516     CMusicInfoTag& tag = *items[i]->GetMusicInfoTag();
 517     CSong song(*items[i]);
 518
 519     // keep the db-only fields intact on rescan...
 520     if (songsMap != NULL)
 521     {
 522       MAPSONGS::iterator it = songsMap->find(items[i]->GetPath());
 523       if (it != songsMap->end())
 524       {
 525         song.iTimesPlayed = it->second.iTimesPlayed;
 526         song.lastPlayed = it->second.lastPlayed;
 527         song.iKaraokeNumber = it->second.iKaraokeNumber;
 528         if (song.rating == '0')    song.rating = it->second.rating;
 529         if (song.strThumb.empty()) song.strThumb = it->second.strThumb;
 530       }
 531     }
 532
 533     if (!tag.GetMusicBrainzAlbumID().empty())
 534     {
 535       VECALBUMS::iterator it;
 536       for (it = albums.begin(); it != albums.end(); ++it)
 537         if (it->strMusicBrainzAlbumID.Equals(tag.GetMusicBrainzAlbumID()))
 538           break;
 539
 540       if (it == albums.end())
 541       {
 542         CAlbum album(*items[i]);
 543         album.songs.push_back(song);
 544         albums.push_back(album);
 545       }
 546       else
 547         it->songs.push_back(song);
 548     }
 549     else
 550       songsByAlbumNames[tag.GetAlbum()].push_back(song);
 551   }
 552
 553   /*
 554    Step 2: Split into unique albums based on album name and album artist
 555    In the case where the album artist is unknown, we use the primary artist
 556    (i.e. first artist from each song).
 557    */
 558   for (map<string, VECSONGS>::iterator songsByAlbumName = songsByAlbumNames.begin(); songsByAlbumName != songsByAlbumNames.end(); ++songsByAlbumName)
 559   {
 560     VECSONGS &songs = songsByAlbumName->second;
 561     // sort the songs by tracknumber to identify duplicate track numbers
 562     sort(songs.begin(), songs.end(), SortSongsByTrack);
 563
 564     // map the songs to their primary artists
 565     bool tracksOverlap = false;
 566     bool hasAlbumArtist = false;
 567     bool isCompilation = true;
 568
 569     map<string, vector<CSong *> > artists;
 570     for (VECSONGS::iterator song = songs.begin(); song != songs.end(); ++song)
 571     {
 572       // test for song overlap
 573       if (song != songs.begin() && song->iTrack == (song - 1)->iTrack)
 574         tracksOverlap = true;
 575
 576       if (!song->bCompilation)
 577         isCompilation = false;
 578
 579       // get primary artist
 580       string primary;
 581       if (!song->albumArtist.empty())
 582       {
 583         primary = song->albumArtist[0];
 584         hasAlbumArtist = true;
 585       }
 586       else if (!song->artist.empty())
 587         primary = song->artist[0];
 588
 589       // add to the artist map
 590       artists[primary].push_back(&(*song));
 591     }
 592
 593     /*
 594      We have a compilation if
 595      1. album name is non-empty AND
 596      2a. no tracks overlap OR
 597      2b. all tracks are marked as part of compilation AND
 598      3a. a unique primary artist is specified as "various" or "various artists" OR
 599      3b. we have at least two primary artists and no album artist specified.
 600      */
 601     bool compilation = !songsByAlbumName->first.empty() && (isCompilation || !tracksOverlap); // 1+2b+2a
 602     if (artists.size() == 1)
 603     {
 604       string artist = artists.begin()->first; StringUtils::ToLower(artist);
 605       if (!StringUtils::EqualsNoCase(artist, "various") &&
 606           !StringUtils::EqualsNoCase(artist, "various artists")) // 3a
 607         compilation = false;
 608     }
 609     else if (hasAlbumArtist) // 3b
 610       compilation = false;
 611
 612     if (compilation)
 613     {
 614       CLog::Log(LOGDEBUG, "Album '%s' is a compilation as there's no overlapping tracks and %s", songsByAlbumName->first.c_str(), hasAlbumArtist ? "the album artist is 'Various'" : "there is more than one unique artist");
 615       artists.clear();
 616       std::string various = g_localizeStrings.Get(340); // Various Artists
 617       vector<string> va; va.push_back(various);
 618       for (VECSONGS::iterator song = songs.begin(); song != songs.end(); ++song)
 619       {
 620         song->albumArtist = va;
 621         artists[various].push_back(&(*song));
 622       }
 623     }
 624
 625     /*
 626      Step 3: Find the common albumartist for each song and assign
 627      albumartist to those tracks that don't have it set.
 628      */
 629     for (map<string, vector<CSong *> >::iterator j = artists.begin(); j != artists.end(); ++j)
 630     {
 631       // find the common artist for these songs
 632       vector<CSong *> &artistSongs = j->second;
 633       vector<string> common = artistSongs.front()->albumArtist.empty() ? artistSongs.front()->artist : artistSongs.front()->albumArtist;
 634       for (vector<CSong *>::iterator k = artistSongs.begin() + 1; k != artistSongs.end(); ++k)
 635       {
 636         unsigned int match = 0;
 637         vector<string> &compare = (*k)->albumArtist.empty() ? (*k)->artist : (*k)->albumArtist;
 638         for (; match < common.size() && match < compare.size(); match++)
 639         {
 640           if (compare[match] != common[match])
 641             break;
 642         }
 643         common.erase(common.begin() + match, common.end());
 644       }
 645
 646       /*
 647        Step 4: Assign the album artist for each song that doesn't have it set
 648        and add to the album vector
 649        */
 650       CAlbum album;
 651       album.strAlbum = songsByAlbumName->first;
 652       album.artist = common;
 653       for (vector<string>::iterator it = common.begin(); it != common.end(); ++it)
 654       {
 655         CStdString strJoinPhrase = (it == --common.end() ? "" : g_advancedSettings.m_musicItemSeparator);
 656         CArtistCredit artistCredit(*it, strJoinPhrase);
 657         album.artistCredits.push_back(artistCredit);
 658       }
 659       album.bCompilation = compilation;
 660       for (vector<CSong *>::iterator k = artistSongs.begin(); k != artistSongs.end(); ++k)
 661       {
 662         if ((*k)->albumArtist.empty())
 663           (*k)->albumArtist = common;
 664         // TODO: in future we may wish to union up the genres, for now we assume they're the same
 665         album.genre = (*k)->genre;
 666         //       in addition, we may want to use year as discriminating for albums
 667         album.iYear = (*k)->iYear;
 668         album.songs.push_back(**k);
 669       }
 670       albums.push_back(album);
 671     }
 672   }
 673 }
 674
 675 int CMusicInfoScanner::RetrieveMusicInfo(const CStdString& strDirectory, CFileItemList& items)
 676 {
 677   MAPSONGS songsMap;
 678
 679   // get all information for all files in current directory from database, and remove them
 680   if (m_musicDatabase.RemoveSongsFromPath(strDirectory, songsMap))
 681     m_needsCleanup = true;
 682
 683   CFileItemList scannedItems;
 684   if (ScanTags(items, scannedItems) == INFO_CANCELLED)
 685     return 0;
 686
 687   VECALBUMS albums;
 688   FileItemsToAlbums(scannedItems, albums, &songsMap);
 689   FindArtForAlbums(albums, items.GetPath());
 690
 691   int numAdded = 0;
 692   ADDON::AddonPtr addon;
 693   ADDON::ScraperPtr albumScraper;
 694   ADDON::ScraperPtr artistScraper;
 695   if(ADDON::CAddonMgr::Get().GetDefault(ADDON::ADDON_SCRAPER_ALBUMS, addon))
 696     albumScraper = boost::dynamic_pointer_cast<ADDON::CScraper>(addon);
 697
 698   if(ADDON::CAddonMgr::Get().GetDefault(ADDON::ADDON_SCRAPER_ARTISTS, addon))
 699     artistScraper = boost::dynamic_pointer_cast<ADDON::CScraper>(addon);
 700
 701   // Add each album
 702   for (VECALBUMS::iterator album = albums.begin(); album != albums.end(); ++album)
 703   {
 704     if (m_bStop)
 705       break;
 706
 707     album->strPath = strDirectory;
 708     m_musicDatabase.BeginTransaction();
 709
 710     // Check if the album has already been downloaded or failed
 711     map<CAlbum, CAlbum>::iterator cachedAlbum = m_albumCache.find(*album);
 712     if (cachedAlbum == m_albumCache.end())
 713     {
 714       // No - download the information
 715       CMusicAlbumInfo albumInfo;
 716       INFO_RET albumDownloadStatus = INFO_NOT_FOUND;
 717       if ((m_flags & SCAN_ONLINE) && albumScraper)
 718         albumDownloadStatus = DownloadAlbumInfo(*album, albumScraper, albumInfo);
 719
 720       if (albumDownloadStatus == INFO_ADDED || albumDownloadStatus == INFO_HAVE_ALREADY)
 721       {
 722         CAlbum &downloadedAlbum = albumInfo.GetAlbum();
 723         downloadedAlbum.idAlbum = m_musicDatabase.AddAlbum(downloadedAlbum.strAlbum,
 724                                                            downloadedAlbum.strMusicBrainzAlbumID,
 725                                                            downloadedAlbum.GetArtistString(),
 726                                                            downloadedAlbum.GetGenreString(),
 727                                                            downloadedAlbum.iYear,
 728                                                            downloadedAlbum.bCompilation);
 729         m_musicDatabase.SetAlbumInfo(downloadedAlbum.idAlbum,
 730                                      downloadedAlbum,
 731                                      downloadedAlbum.songs);
 732         m_musicDatabase.SetArtForItem(downloadedAlbum.idAlbum,
 733                                       "album", album->art);
 734         GetAlbumArtwork(downloadedAlbum.idAlbum, downloadedAlbum);
 735         m_albumCache.insert(make_pair(*album, albumInfo.GetAlbum()));
 736       }
 737       else if (albumDownloadStatus == INFO_CANCELLED)
 738         break;
 739       else
 740       {
 741         // No download info, fallback to already gathered (eg. local) information/art (if any)
 742         album->idAlbum = m_musicDatabase.AddAlbum(album->strAlbum,
 743                                                   album->strMusicBrainzAlbumID,
 744                                                   album->GetArtistString(),
 745                                                   album->GetGenreString(),
 746                                                   album->iYear,
 747                                                   album->bCompilation);
 748         if (!album->art.empty())
 749           m_musicDatabase.SetArtForItem(album->idAlbum,
 750                                         "album", album->art);
 751         m_albumCache.insert(make_pair(*album, *album));
 752       }
 753
 754       // Update the cache pointer with our newly created info
 755       cachedAlbum = m_albumCache.find(*album);
 756     }
 757
 758     if (m_bStop)
 759       break;
 760
 761     // Add the album artists
 762     for (VECARTISTCREDITS::iterator artistCredit = cachedAlbum->second.artistCredits.begin(); artistCredit != cachedAlbum->second.artistCredits.end(); ++artistCredit)
 763     {
 764       if (m_bStop)
 765         break;
 766
 767       // Check if the artist has already been downloaded or failed
 768       map<CArtistCredit, CArtist>::iterator cachedArtist = m_artistCache.find(*artistCredit);
 769       if (cachedArtist == m_artistCache.end())
 770       {
 771         CArtist artistTmp;
 772         artistTmp.strArtist = artistCredit->GetArtist();
 773         artistTmp.strMusicBrainzArtistID = artistCredit->GetMusicBrainzArtistID();
 774         URIUtils::GetParentPath(album->strPath, artistTmp.strPath);
 775
 776         // No - download the information
 777         CMusicArtistInfo artistInfo;
 778         INFO_RET artistDownloadStatus = INFO_NOT_FOUND;
 779         if ((m_flags & SCAN_ONLINE) && artistScraper)
 780           artistDownloadStatus = DownloadArtistInfo(artistTmp, artistScraper, artistInfo);
 781
 782         if (artistDownloadStatus == INFO_ADDED || artistDownloadStatus == INFO_HAVE_ALREADY)
 783         {
 784           CArtist &downloadedArtist = artistInfo.GetArtist();
 785           downloadedArtist.idArtist = m_musicDatabase.AddArtist(downloadedArtist.strArtist,
 786                                                                 downloadedArtist.strMusicBrainzArtistID);
 787           m_musicDatabase.SetArtistInfo(downloadedArtist.idArtist,
 788                                         downloadedArtist);
 789
 790           URIUtils::GetParentPath(album->strPath, downloadedArtist.strPath);
 791           map<string, string> artwork = GetArtistArtwork(downloadedArtist);
 792           // check thumb stuff
 793           m_musicDatabase.SetArtForItem(downloadedArtist.idArtist, "artist", artwork);
 794           m_artistCache.insert(make_pair(*artistCredit, downloadedArtist));
 795         }
 796         else if (artistDownloadStatus == INFO_CANCELLED)
 797           break;
 798         else
 799         {
 800           // Cache the lookup failure so we don't retry
 801           artistTmp.idArtist = m_musicDatabase.AddArtist(artistCredit->GetArtist(), artistCredit->GetMusicBrainzArtistID());
 802           m_artistCache.insert(make_pair(*artistCredit, artistTmp));
 803         }
 804
 805         // Update the cache pointer with our newly created info
 806         cachedArtist = m_artistCache.find(*artistCredit);
 807       }
 808
 809       m_musicDatabase.AddAlbumArtist(cachedArtist->second.idArtist,
 810                                      cachedAlbum->second.idAlbum,
 811                                      artistCredit->GetJoinPhrase(),
 812                                      artistCredit == cachedAlbum->second.artistCredits.begin() ? false : true,
 813                                      std::distance(cachedAlbum->second.artistCredits.begin(), artistCredit));
 814     }
 815
 816     if (m_bStop)
 817       break;
 818
 819     for (VECSONGS::iterator song = album->songs.begin(); song != album->songs.end(); ++song)
 820     {
 821       song->idAlbum = cachedAlbum->second.idAlbum;
 822       song->idSong = m_musicDatabase.AddSong(cachedAlbum->second.idAlbum,
 823                                              song->strTitle, song->strMusicBrainzTrackID,
 824                                              song->strFileName, song->strComment,
 825                                              song->strThumb,
 826                                              song->artist, song->genre,
 827                                              song->iTrack, song->iDuration, song->iYear,
 828                                              song->iTimesPlayed, song->iStartOffset,
 829                                              song->iEndOffset,
 830                                              song->lastPlayed,
 831                                              song->rating,
 832                                              song->iKaraokeNumber);
 833       for (VECARTISTCREDITS::iterator artistCredit = song->artistCredits.begin(); artistCredit != song->artistCredits.end(); ++artistCredit)
 834       {
 835         if (m_bStop)
 836           break;
 837
 838         // Check if the artist has already been downloaded or failed
 839         map<CArtistCredit, CArtist>::iterator cachedArtist = m_artistCache.find(*artistCredit);
 840         if (cachedArtist == m_artistCache.end())
 841         {
 842           CArtist artistTmp;
 843           artistTmp.strArtist = artistCredit->GetArtist();
 844           artistTmp.strMusicBrainzArtistID = artistCredit->GetMusicBrainzArtistID();
 845           URIUtils::GetParentPath(album->strPath, artistTmp.strPath); // FIXME
 846
 847           // No - download the information
 848           CMusicArtistInfo artistInfo;
 849           INFO_RET artistDownloadStatus = INFO_NOT_FOUND;
 850           if ((m_flags & SCAN_ONLINE) && artistScraper)
 851             artistDownloadStatus = DownloadArtistInfo(artistTmp, artistScraper, artistInfo);
 852
 853           if (artistDownloadStatus == INFO_ADDED || artistDownloadStatus == INFO_HAVE_ALREADY)
 854           {
 855             CArtist &downloadedArtist = artistInfo.GetArtist();
 856             downloadedArtist.idArtist = m_musicDatabase.AddArtist(downloadedArtist.strArtist,
 857                                                                   downloadedArtist.strMusicBrainzArtistID);
 858             m_musicDatabase.SetArtistInfo(downloadedArtist.idArtist,
 859                                           downloadedArtist);
 860             // check thumb stuff
 861             URIUtils::GetParentPath(album->strPath, downloadedArtist.strPath);
 862             map<string, string> artwork = GetArtistArtwork(downloadedArtist);
 863             m_musicDatabase.SetArtForItem(downloadedArtist.idArtist, "artist", artwork);
 864             m_artistCache.insert(make_pair(*artistCredit, downloadedArtist));
 865           }
 866           else if (artistDownloadStatus == INFO_CANCELLED)
 867             break;
 868           else
 869           {
 870             // Cache the lookup failure so we don't retry
 871             artistTmp.idArtist = m_musicDatabase.AddArtist(artistCredit->GetArtist(), artistCredit->GetMusicBrainzArtistID());
 872             m_artistCache.insert(make_pair(*artistCredit, artistTmp));
 873           }
 874
 875           // Update the cache pointer with our newly created info
 876           cachedArtist = m_artistCache.find(*artistCredit);
 877         }
 878
 879         m_musicDatabase.AddSongArtist(cachedArtist->second.idArtist,
 880                                       song->idSong,
 881                                       g_advancedSettings.m_musicItemSeparator, // we don't have song artist breakdowns from scrapers, yet
 882                                       artistCredit == song->artistCredits.begin() ? false : true,
 883                                       std::distance(song->artistCredits.begin(), artistCredit));
 884       }
 885     }
 886
 887     if (m_bStop)
 888       break;
 889
 890     // Commit the album to the DB
 891     m_musicDatabase.CommitTransaction();
 892     numAdded += album->songs.size();
 893   }
 894
 895   if (m_bStop)
 896     m_musicDatabase.RollbackTransaction();
 897
 898   if (m_handle)
 899     m_handle->SetTitle(g_localizeStrings.Get(505));
 900
 901   return numAdded;
 902 }
 903
 904 void CMusicInfoScanner::FindArtForAlbums(VECALBUMS &albums, const CStdString &path)
 905 {
 906   /*
 907    If there's a single album in the folder, then art can be taken from
 908    the folder art.
 909    */
 910   std::string albumArt;
 911   if (albums.size() == 1)
 912   {
 913     CFileItem album(path, true);
 914     albumArt = album.GetUserMusicThumb(true);
 915     if (!albumArt.empty())
 916       albums[0].art["thumb"] = albumArt;
 917   }
 918   for (VECALBUMS::iterator i = albums.begin(); i != albums.end(); ++i)
 919   {
 920     CAlbum &album = *i;
 921
 922     if (albums.size() != 1)
 923       albumArt = "";
 924
 925     /*
 926      Find art that is common across these items
 927      If we find a single art image we treat it as the album art
 928      and discard song art else we use first as album art and
 929      keep everything as song art.
 930      */
 931     bool singleArt = true;
 932     CSong *art = NULL;
 933     for (VECSONGS::iterator k = album.songs.begin(); k != album.songs.end(); ++k)
 934     {
 935       CSong &song = *k;
 936       if (song.HasArt())
 937       {
 938         if (art && !art->ArtMatches(song))
 939         {
 940           singleArt = false;
 941           break;
 942         }
 943         if (!art)
 944           art = &song;
 945       }
 946     }
 947
 948     /*
 949       assign the first art found to the album - better than no art at all
 950     */
 951
 952     if (art && albumArt.empty())
 953     {
 954       if (!art->strThumb.empty())
 955         albumArt = art->strThumb;
 956       else
 957         albumArt = CTextureUtils::GetWrappedImageURL(art->strFileName, "music");
 958     }
 959
 960     if (!albumArt.empty())
 961       album.art["thumb"] = albumArt;
 962
 963     if (singleArt)
 964     { //if singleArt then we can clear the artwork for all songs
 965       for (VECSONGS::iterator k = album.songs.begin(); k != album.songs.end(); ++k)
 966         k->strThumb.clear();
 967     }
 968     else
 969     { // more than one piece of art was found for these songs, so cache per song
 970       for (VECSONGS::iterator k = album.songs.begin(); k != album.songs.end(); ++k)
 971       {
 972         if (k->strThumb.empty() && !k->embeddedArt.empty())
 973           k->strThumb = CTextureUtils::GetWrappedImageURL(k->strFileName, "music");
 974       }
 975     }
 976   }
 977   if (albums.size() == 1 && !albumArt.empty())
 978   {
 979     // assign to folder thumb as well
 980     CFileItem albumItem(path, true);
 981     CMusicThumbLoader loader;
 982     loader.SetCachedImage(albumItem, "thumb", albumArt);
 983   }
 984 }
 985
 986 int CMusicInfoScanner::GetPathHash(const CFileItemList &items, CStdString &hash)
 987 {
 988   // Create a hash based on the filenames, filesize and filedate.  Also count the number of files
 989   if (0 == items.Size()) return 0;
 990   XBMC::XBMC_MD5 md5state;
 991   int count = 0;
 992   for (int i = 0; i < items.Size(); ++i)
 993   {
 994     const CFileItemPtr pItem = items[i];
 995     md5state.append(pItem->GetPath());
 996     md5state.append((unsigned char *)&pItem->m_dwSize, sizeof(pItem->m_dwSize));
 997     FILETIME time = pItem->m_dateTime;
 998     md5state.append((unsigned char *)&time, sizeof(FILETIME));
 999     if (pItem->IsAudio() && !pItem->IsPlayList() && !pItem->IsNFO())
1000       count++;
1001   }
1002   md5state.getDigest(hash);
1003   return count;
1004 }
1005
1006 INFO_RET CMusicInfoScanner::UpdateDatabaseAlbumInfo(const CStdString& strPath, CMusicAlbumInfo& albumInfo, bool bAllowSelection, CGUIDialogProgress* pDialog /* = NULL */)
1007 {
1008   m_musicDatabase.Open();
1009   CQueryParams params;
1010   CDirectoryNode::GetDatabaseInfo(strPath, params);
1011
1012   if (params.GetAlbumId() == -1)
1013     return INFO_ERROR;
1014
1015   CAlbum album;
1016   m_musicDatabase.GetAlbumInfo(params.GetAlbumId(), album, &album.songs);
1017
1018   // find album info
1019   ADDON::ScraperPtr scraper;
1020   bool result = m_musicDatabase.GetScraperForPath(strPath, scraper, ADDON::ADDON_SCRAPER_ALBUMS);
1021
1022   m_musicDatabase.Close();
1023
1024   if (!result || !scraper)
1025     return INFO_ERROR;
1026
1027 loop:
1028   CLog::Log(LOGDEBUG, "%s downloading info for: %s", __FUNCTION__, album.strAlbum.c_str());
1029   INFO_RET albumDownloadStatus = DownloadAlbumInfo(album, scraper, albumInfo, pDialog);
1030   if (albumDownloadStatus == INFO_NOT_FOUND)
1031   {
1032     if (pDialog && bAllowSelection)
1033     {
1034       if (!CGUIKeyboardFactory::ShowAndGetInput(album.strAlbum, g_localizeStrings.Get(16011), false))
1035         return INFO_CANCELLED;
1036
1037       CStdString strTempArtist(StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
1038       if (!CGUIKeyboardFactory::ShowAndGetInput(strTempArtist, g_localizeStrings.Get(16025), false))
1039         return INFO_CANCELLED;
1040
1041       album.artist = StringUtils::Split(strTempArtist, g_advancedSettings.m_musicItemSeparator);
1042       goto loop;
1043     }
1044   }
1045   else if (albumDownloadStatus == INFO_ADDED)
1046   {
1047     m_musicDatabase.Open();
1048     m_musicDatabase.SetAlbumInfo(params.GetAlbumId(), albumInfo.GetAlbum(), albumInfo.GetAlbum().songs);
1049     GetAlbumArtwork(params.GetAlbumId(), albumInfo.GetAlbum());
1050     albumInfo.SetLoaded(true);
1051     m_musicDatabase.Close();
1052   }
1053   return albumDownloadStatus;
1054 }
1055
1056 INFO_RET CMusicInfoScanner::UpdateDatabaseArtistInfo(const CStdString& strPath, CMusicArtistInfo& artistInfo, bool bAllowSelection, CGUIDialogProgress* pDialog /* = NULL */)
1057 {
1058   m_musicDatabase.Open();
1059   CQueryParams params;
1060   CDirectoryNode::GetDatabaseInfo(strPath, params);
1061
1062   if (params.GetArtistId() == -1)
1063     return INFO_ERROR;
1064
1065   CArtist artist;
1066   m_musicDatabase.GetArtistInfo(params.GetArtistId(), artist);
1067
1068   // find album info
1069   ADDON::ScraperPtr scraper;
1070   if (!m_musicDatabase.GetScraperForPath(strPath, scraper, ADDON::ADDON_SCRAPER_ARTISTS) || !scraper)
1071     return INFO_ERROR;
1072   m_musicDatabase.Close();
1073
1074 loop:
1075   CLog::Log(LOGDEBUG, "%s downloading info for: %s", __FUNCTION__, artist.strArtist.c_str());
1076   INFO_RET artistDownloadStatus = DownloadArtistInfo(artist, scraper, artistInfo, pDialog);
1077   if (artistDownloadStatus == INFO_NOT_FOUND)
1078   {
1079     if (pDialog && bAllowSelection)
1080     {
1081       if (!CGUIKeyboardFactory::ShowAndGetInput(artist.strArtist, g_localizeStrings.Get(16025), false))
1082         return INFO_CANCELLED;
1083       goto loop;
1084     }
1085   }
1086   else if (artistDownloadStatus == INFO_ADDED)
1087   {
1088     m_musicDatabase.Open();
1089     m_musicDatabase.SetArtistInfo(params.GetArtistId(), artistInfo.GetArtist());
1090     m_musicDatabase.GetArtistPath(params.GetArtistId(), artist.strPath);
1091     map<string, string> artwork = GetArtistArtwork(artist);
1092     m_musicDatabase.SetArtForItem(params.GetArtistId(), "artist", artwork);
1093     artistInfo.SetLoaded();
1094     m_musicDatabase.Close();
1095   }
1096   return artistDownloadStatus;
1097 }
1098
1099 #define THRESHOLD .95f
1100
1101 INFO_RET CMusicInfoScanner::DownloadAlbumInfo(const CAlbum& album, ADDON::ScraperPtr& info, CMusicAlbumInfo& albumInfo, CGUIDialogProgress* pDialog)
1102 {
1103   if (m_handle)
1104   {
1105     m_handle->SetTitle(StringUtils::Format(g_localizeStrings.Get(20321), info->Name().c_str()));
1106     m_handle->SetText(StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator) + " - " + album.strAlbum);
1107   }
1108
1109   // clear our scraper cache
1110   info->ClearCache();
1111
1112   CMusicInfoScraper scraper(info);
1113   bool bMusicBrainz = false;
1114   if (!album.strMusicBrainzAlbumID.empty())
1115   {
1116     CScraperUrl musicBrainzURL;
1117     if (ResolveMusicBrainz(album.strMusicBrainzAlbumID, info, scraper, musicBrainzURL))
1118     {
1119       CMusicAlbumInfo albumNfo("nfo", musicBrainzURL);
1120       scraper.GetAlbums().clear();
1121       scraper.GetAlbums().push_back(albumNfo);
1122       bMusicBrainz = true;
1123     }
1124   }
1125
1126   // handle nfo files
1127   CStdString strNfo = URIUtils::AddFileToFolder(album.strPath, "album.nfo");
1128   CNfoFile::NFOResult result = CNfoFile::NO_NFO;
1129   CNfoFile nfoReader;
1130   if (XFILE::CFile::Exists(strNfo))
1131   {
1132     CLog::Log(LOGDEBUG,"Found matching nfo file: %s", strNfo.c_str());
1133     result = nfoReader.Create(strNfo, info, -1, album.strPath);
1134     if (result == CNfoFile::FULL_NFO)
1135     {
1136       CLog::Log(LOGDEBUG, "%s Got details from nfo", __FUNCTION__);
1137       nfoReader.GetDetails(albumInfo.GetAlbum());
1138       return INFO_ADDED;
1139     }
1140     else if (result == CNfoFile::URL_NFO || result == CNfoFile::COMBINED_NFO)
1141     {
1142       CScraperUrl scrUrl(nfoReader.ScraperUrl());
1143       CMusicAlbumInfo albumNfo("nfo",scrUrl);
1144       info = nfoReader.GetScraperInfo();
1145       CLog::Log(LOGDEBUG,"-- nfo-scraper: %s",info->Name().c_str());
1146       CLog::Log(LOGDEBUG,"-- nfo url: %s", scrUrl.m_url[0].m_url.c_str());
1147       scraper.SetScraperInfo(info);
1148       scraper.GetAlbums().clear();
1149       scraper.GetAlbums().push_back(albumNfo);
1150     }
1151     else
1152       CLog::Log(LOGERROR,"Unable to find an url in nfo file: %s", strNfo.c_str());
1153   }
1154
1155   if (!scraper.CheckValidOrFallback(CSettings::Get().GetString("musiclibrary.albumsscraper")))
1156   { // the current scraper is invalid, as is the default - bail
1157     CLog::Log(LOGERROR, "%s - current and default scrapers are invalid.  Pick another one", __FUNCTION__);
1158     return INFO_ERROR;
1159   }
1160
1161   if (!scraper.GetAlbumCount())
1162   {
1163     scraper.FindAlbumInfo(album.strAlbum, StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
1164
1165     while (!scraper.Completed())
1166     {
1167       if (m_bStop)
1168       {
1169         scraper.Cancel();
1170         return INFO_CANCELLED;
1171       }
1172       Sleep(1);
1173     }
1174   }
1175
1176   CGUIDialogSelect *pDlg = NULL;
1177   int iSelectedAlbum=0;
1178   if (result == CNfoFile::NO_NFO && !bMusicBrainz)
1179   {
1180     iSelectedAlbum = -1; // set negative so that we can detect a failure
1181     if (scraper.Succeeded() && scraper.GetAlbumCount() >= 1)
1182     {
1183       double bestRelevance = 0;
1184       double minRelevance = THRESHOLD;
1185       if (scraper.GetAlbumCount() > 1) // score the matches
1186       {
1187         //show dialog with all albums found
1188         if (pDialog)
1189         {
1190           pDlg = (CGUIDialogSelect*)g_windowManager.GetWindow(WINDOW_DIALOG_SELECT);
1191           pDlg->SetHeading(g_localizeStrings.Get(181).c_str());
1192           pDlg->Reset();
1193           pDlg->EnableButton(true, 413); // manual
1194         }
1195
1196         for (int i = 0; i < scraper.GetAlbumCount(); ++i)
1197         {
1198           CMusicAlbumInfo& info = scraper.GetAlbum(i);
1199           double relevance = info.GetRelevance();
1200           if (relevance < 0)
1201             relevance = CUtil::AlbumRelevance(info.GetAlbum().strAlbum, album.strAlbum, StringUtils::Join(info.GetAlbum().artist, g_advancedSettings.m_musicItemSeparator), StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
1202
1203           // if we're doing auto-selection (ie querying all albums at once, then allow 95->100% for perfect matches)
1204           // otherwise, perfect matches only
1205           if (relevance >= max(minRelevance, bestRelevance))
1206           { // we auto-select the best of these
1207             bestRelevance = relevance;
1208             iSelectedAlbum = i;
1209           }
1210           if (pDialog)
1211           {
1212             // set the label to [relevance]  album - artist
1213             CStdString strTemp;
1214             strTemp.Format("[%0.2f]  %s", relevance, info.GetTitle2());
1215             CFileItem item(strTemp);
1216             item.m_idepth = i; // use this to hold the index of the album in the scraper
1217             pDlg->Add(&item);
1218           }
1219           if (relevance > .99f) // we're so close, no reason to search further
1220             break;
1221         }
1222
1223         if (pDialog && bestRelevance < THRESHOLD)
1224         {
1225           pDlg->Sort(false);
1226           pDlg->DoModal();
1227
1228           // and wait till user selects one
1229           if (pDlg->GetSelectedLabel() < 0)
1230           { // none chosen
1231             if (!pDlg->IsButtonPressed())
1232               return INFO_CANCELLED;
1233
1234             // manual button pressed
1235             CStdString strNewAlbum = album.strAlbum;
1236             if (!CGUIKeyboardFactory::ShowAndGetInput(strNewAlbum, g_localizeStrings.Get(16011), false)) return INFO_CANCELLED;
1237             if (strNewAlbum == "") return INFO_CANCELLED;
1238
1239             CStdString strNewArtist = StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator);
1240             if (!CGUIKeyboardFactory::ShowAndGetInput(strNewArtist, g_localizeStrings.Get(16025), false)) return INFO_CANCELLED;
1241
1242             pDialog->SetLine(0, strNewAlbum);
1243             pDialog->SetLine(1, strNewArtist);
1244             pDialog->Progress();
1245
1246             CAlbum newAlbum = album;
1247             newAlbum.strAlbum = strNewAlbum;
1248             newAlbum.artist = StringUtils::Split(strNewArtist, g_advancedSettings.m_musicItemSeparator);
1249
1250             return DownloadAlbumInfo(newAlbum, info, albumInfo, pDialog);
1251           }
1252           iSelectedAlbum = pDlg->GetSelectedItem()->m_idepth;
1253         }
1254       }
1255       else
1256       {
1257         CMusicAlbumInfo& info = scraper.GetAlbum(0);
1258         double relevance = info.GetRelevance();
1259         if (relevance < 0)
1260           relevance = CUtil::AlbumRelevance(info.GetAlbum().strAlbum,
1261                                             album.strAlbum,
1262                                             StringUtils::Join(info.GetAlbum().artist, g_advancedSettings.m_musicItemSeparator),
1263                                             StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
1264         if (relevance < THRESHOLD)
1265           return INFO_NOT_FOUND;
1266
1267         iSelectedAlbum = 0;
1268       }
1269     }
1270
1271     if (iSelectedAlbum < 0)
1272       return INFO_NOT_FOUND;
1273
1274   }
1275
1276   scraper.LoadAlbumInfo(iSelectedAlbum);
1277   while (!scraper.Completed())
1278   {
1279     if (m_bStop)
1280     {
1281       scraper.Cancel();
1282       return INFO_CANCELLED;
1283     }
1284     Sleep(1);
1285   }
1286
1287   if (!scraper.Succeeded())
1288     return INFO_ERROR;
1289
1290   albumInfo = scraper.GetAlbum(iSelectedAlbum);
1291
1292   if (result == CNfoFile::COMBINED_NFO)
1293     nfoReader.GetDetails(albumInfo.GetAlbum(), NULL, true);
1294
1295   return INFO_ADDED;
1296 }
1297
1298 void CMusicInfoScanner::GetAlbumArtwork(long id, const CAlbum &album)
1299 {
1300   if (album.thumbURL.m_url.size())
1301   {
1302     if (m_musicDatabase.GetArtForItem(id, "album", "thumb").empty())
1303     {
1304       string thumb = CScraperUrl::GetThumbURL(album.thumbURL.GetFirstThumb());
1305       if (!thumb.empty())
1306       {
1307         CTextureCache::Get().BackgroundCacheImage(thumb);
1308         m_musicDatabase.SetArtForItem(id, "album", "thumb", thumb);
1309       }
1310     }
1311   }
1312 }
1313
1314 INFO_RET CMusicInfoScanner::DownloadArtistInfo(const CArtist& artist, ADDON::ScraperPtr& info, MUSIC_GRABBER::CMusicArtistInfo& artistInfo, CGUIDialogProgress* pDialog)
1315 {
1316   if (m_handle)
1317   {
1318     m_handle->SetTitle(StringUtils::Format(g_localizeStrings.Get(20320), info->Name().c_str()));
1319     m_handle->SetText(artist.strArtist);
1320   }
1321
1322   // clear our scraper cache
1323   info->ClearCache();
1324
1325   CMusicInfoScraper scraper(info);
1326   bool bMusicBrainz = false;
1327   if (!artist.strMusicBrainzArtistID.empty())
1328   {
1329     CScraperUrl musicBrainzURL;
1330     if (ResolveMusicBrainz(artist.strMusicBrainzArtistID, info, scraper, musicBrainzURL))
1331     {
1332       CMusicArtistInfo artistNfo("nfo", musicBrainzURL);
1333       scraper.GetArtists().clear();
1334       scraper.GetArtists().push_back(artistNfo);
1335       bMusicBrainz = true;
1336     }
1337   }
1338
1339   // handle nfo files
1340   CStdString strNfo = URIUtils::AddFileToFolder(artist.strPath, "artist.nfo");
1341   CNfoFile::NFOResult result=CNfoFile::NO_NFO;
1342   CNfoFile nfoReader;
1343   if (XFILE::CFile::Exists(strNfo))
1344   {
1345     CLog::Log(LOGDEBUG,"Found matching nfo file: %s", strNfo.c_str());
1346     result = nfoReader.Create(strNfo, info);
1347     if (result == CNfoFile::FULL_NFO)
1348     {
1349       CLog::Log(LOGDEBUG, "%s Got details from nfo", __FUNCTION__);
1350       nfoReader.GetDetails(artistInfo.GetArtist());
1351       return INFO_ADDED;
1352     }
1353     else if (result == CNfoFile::URL_NFO || result == CNfoFile::COMBINED_NFO)
1354     {
1355       CScraperUrl scrUrl(nfoReader.ScraperUrl());
1356       CMusicArtistInfo artistNfo("nfo",scrUrl);
1357       info = nfoReader.GetScraperInfo();
1358       CLog::Log(LOGDEBUG,"-- nfo-scraper: %s",info->Name().c_str());
1359       CLog::Log(LOGDEBUG,"-- nfo url: %s", scrUrl.m_url[0].m_url.c_str());
1360       scraper.SetScraperInfo(info);
1361       scraper.GetArtists().push_back(artistNfo);
1362     }
1363     else
1364       CLog::Log(LOGERROR,"Unable to find an url in nfo file: %s", strNfo.c_str());
1365   }
1366
1367   if (!scraper.GetArtistCount())
1368   {
1369     scraper.FindArtistInfo(artist.strArtist);
1370
1371     while (!scraper.Completed())
1372     {
1373       if (m_bStop)
1374       {
1375         scraper.Cancel();
1376         return INFO_CANCELLED;
1377       }
1378       Sleep(1);
1379     }
1380   }
1381
1382   int iSelectedArtist = 0;
1383   if (result == CNfoFile::NO_NFO && !bMusicBrainz)
1384   {
1385     if (scraper.GetArtistCount() >= 1)
1386     {
1387       // now load the first match
1388       if (pDialog && scraper.GetArtistCount() > 1)
1389       {
1390         // if we found more then 1 album, let user choose one
1391         CGUIDialogSelect *pDlg = (CGUIDialogSelect*)g_windowManager.GetWindow(WINDOW_DIALOG_SELECT);
1392         if (pDlg)
1393         {
1394           pDlg->SetHeading(g_localizeStrings.Get(21890));
1395           pDlg->Reset();
1396           pDlg->EnableButton(true, 413); // manual
1397
1398           for (int i = 0; i < scraper.GetArtistCount(); ++i)
1399           {
1400             // set the label to artist
1401             CFileItem item(scraper.GetArtist(i).GetArtist());
1402             CStdString strTemp=scraper.GetArtist(i).GetArtist().strArtist;
1403             if (!scraper.GetArtist(i).GetArtist().strBorn.IsEmpty())
1404               strTemp += " ("+scraper.GetArtist(i).GetArtist().strBorn+")";
1405             if (!scraper.GetArtist(i).GetArtist().genre.empty())
1406             {
1407               CStdString genres = StringUtils::Join(scraper.GetArtist(i).GetArtist().genre, g_advancedSettings.m_musicItemSeparator);
1408               if (!genres.empty())
1409                 strTemp.Format("[%s] %s", genres.c_str(), strTemp.c_str());
1410             }
1411             item.SetLabel(strTemp);
1412             item.m_idepth = i; // use this to hold the index of the album in the scraper
1413             pDlg->Add(&item);
1414           }
1415           pDlg->DoModal();
1416
1417           // and wait till user selects one
1418           if (pDlg->GetSelectedLabel() < 0)
1419           { // none chosen
1420             if (!pDlg->IsButtonPressed())
1421               return INFO_CANCELLED;
1422
1423             // manual button pressed
1424             CStdString strNewArtist = artist.strArtist;
1425             if (!CGUIKeyboardFactory::ShowAndGetInput(strNewArtist, g_localizeStrings.Get(16025), false)) return INFO_CANCELLED;
1426
1427             if (pDialog)
1428             {
1429               pDialog->SetLine(0, strNewArtist);
1430               pDialog->Progress();
1431             }
1432
1433             CArtist newArtist;
1434             newArtist.strArtist = strNewArtist;
1435             return DownloadArtistInfo(newArtist, info, artistInfo, pDialog);
1436           }
1437           iSelectedArtist = pDlg->GetSelectedItem()->m_idepth;
1438         }
1439       }
1440     }
1441     else
1442       return INFO_NOT_FOUND;
1443   }
1444
1445   scraper.LoadArtistInfo(iSelectedArtist, artist.strArtist);
1446   while (!scraper.Completed())
1447   {
1448     if (m_bStop)
1449     {
1450       scraper.Cancel();
1451       return INFO_CANCELLED;
1452     }
1453     Sleep(1);
1454   }
1455
1456   if (!scraper.Succeeded())
1457     return INFO_ERROR;
1458
1459   artistInfo = scraper.GetArtist(iSelectedArtist);
1460
1461   if (result == CNfoFile::COMBINED_NFO)
1462     nfoReader.GetDetails(artistInfo.GetArtist(), NULL, true);
1463
1464   return INFO_ADDED;
1465 }
1466
1467 bool CMusicInfoScanner::ResolveMusicBrainz(const CStdString strMusicBrainzID, ScraperPtr &preferredScraper, CMusicInfoScraper &musicInfoScraper, CScraperUrl &musicBrainzURL)
1468 {
1469   // We have a MusicBrainz ID
1470   // Get a scraper that can resolve it to a MusicBrainz URL & force our
1471   // search directly to the specific album.
1472   bool bMusicBrainz = false;
1473   ADDON::TYPE type = ScraperTypeFromContent(preferredScraper->Content());
1474
1475   CFileItemList items;
1476   ADDON::AddonPtr addon;
1477   ADDON::ScraperPtr defaultScraper;
1478   if (ADDON::CAddonMgr::Get().GetDefault(type, addon))
1479     defaultScraper = boost::dynamic_pointer_cast<CScraper>(addon);
1480
1481   vector<ScraperPtr> vecScrapers;
1482
1483   // add selected scraper - first proirity
1484   if (preferredScraper)
1485     vecScrapers.push_back(preferredScraper);
1486
1487   // Add all scrapers except selected and default
1488   VECADDONS addons;
1489   CAddonMgr::Get().GetAddons(type, addons);
1490
1491   for (unsigned i = 0; i < addons.size(); ++i)
1492   {
1493     ScraperPtr scraper = boost::dynamic_pointer_cast<CScraper>(addons[i]);
1494
1495     // skip if scraper requires settings and there's nothing set yet
1496     if (!scraper || (scraper->RequiresSettings() && !scraper->HasUserSettings()))
1497       continue;
1498
1499     if((!preferredScraper || preferredScraper->ID() != scraper->ID()) && (!defaultScraper || defaultScraper->ID() != scraper->ID()) )
1500       vecScrapers.push_back(scraper);
1501   }
1502
1503   // add default scraper - not user selectable so it's last priority
1504   if(defaultScraper &&
1505      (!preferredScraper || preferredScraper->ID() != defaultScraper->ID()) &&
1506      (!defaultScraper->RequiresSettings() || defaultScraper->HasUserSettings()))
1507     vecScrapers.push_back(defaultScraper);
1508
1509   for (unsigned int i=0; i < vecScrapers.size(); ++i)
1510   {
1511     if (vecScrapers[i]->Type() != type)
1512       continue;
1513
1514     vecScrapers[i]->ClearCache();
1515     try
1516     {
1517       musicBrainzURL = vecScrapers[i]->ResolveIDToUrl(strMusicBrainzID);
1518     }
1519     catch (const ADDON::CScraperError &sce)
1520     {
1521       if (!sce.FAborted())
1522         continue;
1523     }
1524     if (!musicBrainzURL.m_url.empty())
1525     {
1526       Sleep(2000); // MusicBrainz rate-limits queries to 1 p.s - once we hit the rate-limiter
1527                    // they start serving up the 'you hit the rate-limiter' page fast - meaning
1528                    // we will never get below the rate-limit threshold again in a specific run.
1529                    // This helps us to avoidthe rate-limiter as far as possible.
1530       CLog::Log(LOGDEBUG,"-- nfo-scraper: %s",vecScrapers[i]->Name().c_str());
1531       CLog::Log(LOGDEBUG,"-- nfo url: %s", musicBrainzURL.m_url[0].m_url.c_str());
1532       musicInfoScraper.SetScraperInfo(vecScrapers[i]);
1533       bMusicBrainz = true;
1534       break;
1535     }
1536   }
1537
1538   return bMusicBrainz;
1539 }
1540
1541 map<string, string> CMusicInfoScanner::GetArtistArtwork(const CArtist& artist)
1542 {
1543   map<string, string> artwork;
1544
1545   // check thumb
1546   CStdString strFolder;
1547   CStdString thumb;
1548   if (!artist.strPath.IsEmpty())
1549   {
1550     strFolder = artist.strPath;
1551     for (int i = 0; i < 3 && thumb.IsEmpty(); ++i)
1552     {
1553       CFileItem item(strFolder, true);
1554       thumb = item.GetUserMusicThumb(true);
1555       strFolder = URIUtils::GetParentPath(strFolder);
1556     }
1557   }
1558   if (thumb.IsEmpty())
1559     thumb = CScraperUrl::GetThumbURL(artist.thumbURL.GetFirstThumb());
1560   if (!thumb.IsEmpty())
1561   {
1562     CTextureCache::Get().BackgroundCacheImage(thumb);
1563     artwork.insert(make_pair("thumb", thumb));
1564   }
1565
1566   // check fanart
1567   CStdString fanart;
1568   if (!artist.strPath.IsEmpty())
1569   {
1570     strFolder = artist.strPath;
1571     for (int i = 0; i < 3 && fanart.IsEmpty(); ++i)
1572     {
1573       CFileItem item(strFolder, true);
1574       fanart = item.GetLocalFanart();
1575       strFolder = URIUtils::GetParentPath(strFolder);
1576     }
1577   }
1578   if (fanart.IsEmpty())
1579     fanart = artist.fanart.GetImageURL();
1580   if (!fanart.IsEmpty())
1581   {
1582     CTextureCache::Get().BackgroundCacheImage(fanart);
1583     artwork.insert(make_pair("fanart", fanart));
1584   }
1585
1586   return artwork;
1587 }
1588
1589 // This function is the Run() function of the IRunnable
1590 // CFileCountReader and runs in a separate thread.
1591 void CMusicInfoScanner::Run()
1592 {
1593   int count = 0;
1594   for (set<std::string>::iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end() && !m_bStop; ++it)
1595   {
1596     count+=CountFilesRecursively(*it);
1597   }
1598   m_itemCount = count;
1599 }
1600
1601 // Recurse through all folders we scan and count files
1602 int CMusicInfoScanner::CountFilesRecursively(const CStdString& strPath)
1603 {
1604   // load subfolder
1605   CFileItemList items;
1606   CDirectory::GetDirectory(strPath, items, g_advancedSettings.m_musicExtensions, DIR_FLAG_NO_FILE_DIRS);
1607
1608   if (m_bStop)
1609     return 0;
1610
1611   // true for recursive counting
1612   int count = CountFiles(items, true);
1613   return count;
1614 }
1615
1616 int CMusicInfoScanner::CountFiles(const CFileItemList &items, bool recursive)
1617 {
1618   int count = 0;
1619   for (int i=0; i<items.Size(); ++i)
1620   {
1621     const CFileItemPtr pItem=items[i];
1622
1623     if (recursive && pItem->m_bIsFolder)
1624       count+=CountFilesRecursively(pItem->GetPath());
1625     else if (pItem->IsAudio() && !pItem->IsPlayList() && !pItem->IsNFO())
1626       count++;
1627   }
1628   return count;
1629 }