code.vuplus.com Git - vuplus_xbmc/blob - xbmc/music/infoscanner/MusicInfoScanner.cpp

   1 /*
   2  *      Copyright (C) 2005-2013 Team XBMC
   3  *      http://xbmc.org
   4  *
   5  *  This Program is free software; you can redistribute it and/or modify
   6  *  it under the terms of the GNU General Public License as published by
   7  *  the Free Software Foundation; either version 2, or (at your option)
   8  *  any later version.
   9  *
  10  *  This Program is distributed in the hope that it will be useful,
  11  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  *  GNU General Public License for more details.
  14  *
  15  *  You should have received a copy of the GNU General Public License
  16  *  along with XBMC; see the file COPYING.  If not, see
  17  *  <http://www.gnu.org/licenses/>.
  18  *
  19  */
  20
  21 #include "threads/SystemClock.h"
  22 #include "MusicInfoScanner.h"
  23 #include "music/tags/MusicInfoTagLoaderFactory.h"
  24 #include "MusicAlbumInfo.h"
  25 #include "MusicInfoScraper.h"
  26 #include "filesystem/MusicDatabaseDirectory.h"
  27 #include "filesystem/MusicDatabaseDirectory/DirectoryNode.h"
  28 #include "Util.h"
  29 #include "utils/md5.h"
  30 #include "GUIInfoManager.h"
  31 #include "utils/Variant.h"
  32 #include "NfoFile.h"
  33 #include "music/tags/MusicInfoTag.h"
  34 #include "guilib/GUIWindowManager.h"
  35 #include "dialogs/GUIDialogExtendedProgressBar.h"
  36 #include "dialogs/GUIDialogProgress.h"
  37 #include "dialogs/GUIDialogSelect.h"
  38 #include "guilib/GUIKeyboardFactory.h"
  39 #include "filesystem/File.h"
  40 #include "filesystem/Directory.h"
  41 #include "settings/AdvancedSettings.h"
  42 #include "settings/Settings.h"
  43 #include "FileItem.h"
  44 #include "guilib/LocalizeStrings.h"
  45 #include "utils/StringUtils.h"
  46 #include "utils/TimeUtils.h"
  47 #include "utils/log.h"
  48 #include "utils/URIUtils.h"
  49 #include "TextureCache.h"
  50 #include "music/MusicThumbLoader.h"
  51 #include "interfaces/AnnouncementManager.h"
  52 #include "GUIUserMessages.h"
  53 #include "addons/AddonManager.h"
  54 #include "addons/Scraper.h"
  55
  56 #include <algorithm>
  57
  58 using namespace std;
  59 using namespace MUSIC_INFO;
  60 using namespace XFILE;
  61 using namespace MUSICDATABASEDIRECTORY;
  62 using namespace MUSIC_GRABBER;
  63 using namespace ADDON;
  64
  65 CMusicInfoScanner::CMusicInfoScanner() : CThread("MusicInfoScanner"), m_fileCountReader(this, "MusicFileCounter")
  66 {
  67   m_bRunning = false;
  68   m_showDialog = false;
  69   m_handle = NULL;
  70   m_bCanInterrupt = false;
  71   m_currentItem=0;
  72   m_itemCount=0;
  73   m_flags = 0;
  74 }
  75
  76 CMusicInfoScanner::~CMusicInfoScanner()
  77 {
  78 }
  79
  80 void CMusicInfoScanner::Process()
  81 {
  82   ANNOUNCEMENT::CAnnouncementManager::Announce(ANNOUNCEMENT::AudioLibrary, "xbmc", "OnScanStarted");
  83   try
  84   {
  85     unsigned int tick = XbmcThreads::SystemClockMillis();
  86
  87     m_musicDatabase.Open();
  88
  89     if (m_showDialog && !CSettings::Get().GetBool("musiclibrary.backgroundupdate"))
  90     {
  91       CGUIDialogExtendedProgressBar* dialog =
  92         (CGUIDialogExtendedProgressBar*)g_windowManager.GetWindow(WINDOW_DIALOG_EXT_PROGRESS);
  93       m_handle = dialog->GetHandle(g_localizeStrings.Get(314));
  94     }
  95
  96     m_bCanInterrupt = true;
  97
  98     if (m_scanType == 0) // load info from files
  99     {
 100       CLog::Log(LOGDEBUG, "%s - Starting scan", __FUNCTION__);
 101
 102       if (m_handle)
 103         m_handle->SetTitle(g_localizeStrings.Get(505));
 104
 105       // Reset progress vars
 106       m_currentItem=0;
 107       m_itemCount=-1;
 108
 109       // Create the thread to count all files to be scanned
 110       SetPriority( GetMinPriority() );
 111       if (m_handle)
 112         m_fileCountReader.Create();
 113
 114       // Database operations should not be canceled
 115       // using Interupt() while scanning as it could
 116       // result in unexpected behaviour.
 117       m_bCanInterrupt = false;
 118       m_needsCleanup = false;
 119
 120       bool commit = false;
 121       bool cancelled = false;
 122       for (std::set<std::string>::const_iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end(); it++)
 123       {
 124         if (!CDirectory::Exists(*it) && !m_bClean)
 125         {
 126           /*
 127            * Note that this will skip scanning (if m_bClean is disabled) if the directory really
 128            * doesn't exist. Since the music scanner is fed with a list of existing paths from the DB
 129            * and cleans out all songs under that path as its first step before re-adding files, if
 130            * the entire source is offline we totally empty the music database in one go.
 131            */
 132           CLog::Log(LOGWARNING, "%s directory '%s' does not exist - skipping scan.", __FUNCTION__, it->c_str());
 133           continue;
 134         }
 135         else if (!DoScan(*it))
 136           cancelled = true;
 137         commit = !cancelled;
 138       }
 139
 140       if (commit)
 141       {
 142         g_infoManager.ResetLibraryBools();
 143
 144         if (m_needsCleanup)
 145         {
 146           if (m_handle)
 147           {
 148             m_handle->SetTitle(g_localizeStrings.Get(700));
 149             m_handle->SetText("");
 150           }
 151
 152           m_musicDatabase.CleanupOrphanedItems();
 153
 154           if (m_handle)
 155             m_handle->SetTitle(g_localizeStrings.Get(331));
 156
 157           m_musicDatabase.Compress(false);
 158         }
 159       }
 160
 161       m_fileCountReader.StopThread();
 162
 163       m_musicDatabase.EmptyCache();
 164
 165       tick = XbmcThreads::SystemClockMillis() - tick;
 166       CLog::Log(LOGNOTICE, "My Music: Scanning for music info using worker thread, operation took %s", StringUtils::SecondsToTimeString(tick / 1000).c_str());
 167     }
 168     if (m_scanType == 1) // load album info
 169     {
 170       for (std::set<std::string>::const_iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end(); ++it)
 171       {
 172         CQueryParams params;
 173         CDirectoryNode::GetDatabaseInfo(*it, params);
 174         if (m_musicDatabase.HasAlbumInfo(params.GetArtistId())) // should this be here?
 175           continue;
 176
 177         CAlbum album;
 178         m_musicDatabase.GetAlbumInfo(params.GetAlbumId(), album, &album.songs);
 179         if (m_handle)
 180         {
 181           float percentage = (float) std::distance(it, m_pathsToScan.end()) / m_pathsToScan.size();
 182           m_handle->SetText(StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator) + " - " + album.strAlbum);
 183           m_handle->SetPercentage(percentage);
 184         }
 185
 186         CMusicAlbumInfo albumInfo;
 187         UpdateDatabaseAlbumInfo(*it, albumInfo, false);
 188
 189         if (m_bStop)
 190           break;
 191       }
 192     }
 193     if (m_scanType == 2) // load artist info
 194     {
 195       for (std::set<std::string>::const_iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end(); ++it)
 196       {
 197         CQueryParams params;
 198         CDirectoryNode::GetDatabaseInfo(*it, params);
 199         if (m_musicDatabase.HasArtistInfo(params.GetArtistId())) // should this be here?
 200             continue;
 201
 202         CArtist artist;
 203         m_musicDatabase.GetArtistInfo(params.GetArtistId(), artist);
 204         m_musicDatabase.GetArtistPath(params.GetArtistId(), artist.strPath);
 205
 206         if (m_handle)
 207         {
 208           float percentage = (float) (std::distance(m_pathsToScan.begin(), it) / m_pathsToScan.size()) * 100;
 209           m_handle->SetText(artist.strArtist);
 210           m_handle->SetPercentage(percentage);
 211         }
 212
 213         CMusicArtistInfo artistInfo;
 214         UpdateDatabaseArtistInfo(*it, artistInfo, false);
 215
 216         if (m_bStop)
 217           break;
 218       }
 219     }
 220
 221   }
 222   catch (...)
 223   {
 224     CLog::Log(LOGERROR, "MusicInfoScanner: Exception while scanning.");
 225   }
 226   m_musicDatabase.Close();
 227   CLog::Log(LOGDEBUG, "%s - Finished scan", __FUNCTION__);
 228
 229   m_bRunning = false;
 230   ANNOUNCEMENT::CAnnouncementManager::Announce(ANNOUNCEMENT::AudioLibrary, "xbmc", "OnScanFinished");
 231
 232   // we need to clear the musicdb cache and update any active lists
 233   CUtil::DeleteMusicDatabaseDirectoryCache();
 234   CGUIMessage msg(GUI_MSG_SCAN_FINISHED, 0, 0, 0);
 235   g_windowManager.SendThreadMessage(msg);
 236
 237   if (m_handle)
 238     m_handle->MarkFinished();
 239   m_handle = NULL;
 240 }
 241
 242 void CMusicInfoScanner::Start(const CStdString& strDirectory, int flags)
 243 {
 244   m_fileCountReader.StopThread();
 245   StopThread();
 246   m_pathsToScan.clear();
 247   m_flags = flags;
 248
 249   if (strDirectory.IsEmpty())
 250   { // scan all paths in the database.  We do this by scanning all paths in the db, and crossing them off the list as
 251     // we go.
 252     m_musicDatabase.Open();
 253     m_musicDatabase.GetPaths(m_pathsToScan);
 254     m_musicDatabase.Close();
 255   }
 256   else
 257     m_pathsToScan.insert(strDirectory);
 258   m_bClean = g_advancedSettings.m_bMusicLibraryCleanOnUpdate;
 259
 260   m_scanType = 0;
 261   Create();
 262   m_bRunning = true;
 263 }
 264
 265 void CMusicInfoScanner::FetchAlbumInfo(const CStdString& strDirectory,
 266                                        bool refresh)
 267 {
 268   m_fileCountReader.StopThread();
 269   StopThread();
 270   m_pathsToScan.clear();
 271
 272   CFileItemList items;
 273   if (strDirectory.IsEmpty())
 274   {
 275     m_musicDatabase.Open();
 276     m_musicDatabase.GetAlbumsNav("musicdb://albums/", items);
 277     m_musicDatabase.Close();
 278   }
 279   else
 280   {
 281     if (URIUtils::HasSlashAtEnd(strDirectory)) // directory
 282       CDirectory::GetDirectory(strDirectory,items);
 283     else
 284     {
 285       CFileItemPtr item(new CFileItem(strDirectory,false));
 286       items.Add(item);
 287     }
 288   }
 289
 290   m_musicDatabase.Open();
 291   for (int i=0;i<items.Size();++i)
 292   {
 293     if (CMusicDatabaseDirectory::IsAllItem(items[i]->GetPath()) || items[i]->IsParentFolder())
 294       continue;
 295
 296     m_pathsToScan.insert(items[i]->GetPath());
 297     if (refresh)
 298     {
 299       m_musicDatabase.DeleteAlbumInfo(items[i]->GetMusicInfoTag()->GetDatabaseId());
 300     }
 301   }
 302   m_musicDatabase.Close();
 303
 304   m_scanType = 1;
 305   Create();
 306   m_bRunning = true;
 307 }
 308
 309 void CMusicInfoScanner::FetchArtistInfo(const CStdString& strDirectory,
 310                                         bool refresh)
 311 {
 312   m_fileCountReader.StopThread();
 313   StopThread();
 314   m_pathsToScan.clear();
 315   CFileItemList items;
 316
 317   if (strDirectory.IsEmpty())
 318   {
 319     m_musicDatabase.Open();
 320     m_musicDatabase.GetArtistsNav("musicdb://artists/", items, false, -1);
 321     m_musicDatabase.Close();
 322   }
 323   else
 324   {
 325     if (URIUtils::HasSlashAtEnd(strDirectory)) // directory
 326       CDirectory::GetDirectory(strDirectory,items);
 327     else
 328     {
 329       CFileItemPtr newItem(new CFileItem(strDirectory,false));
 330       items.Add(newItem);
 331     }
 332   }
 333
 334   m_musicDatabase.Open();
 335   for (int i=0;i<items.Size();++i)
 336   {
 337     if (CMusicDatabaseDirectory::IsAllItem(items[i]->GetPath()) || items[i]->IsParentFolder())
 338       continue;
 339
 340     m_pathsToScan.insert(items[i]->GetPath());
 341     if (refresh)
 342     {
 343       m_musicDatabase.DeleteArtistInfo(items[i]->GetMusicInfoTag()->GetDatabaseId());
 344     }
 345   }
 346   m_musicDatabase.Close();
 347
 348   m_scanType = 2;
 349   Create();
 350   m_bRunning = true;
 351 }
 352
 353 bool CMusicInfoScanner::IsScanning()
 354 {
 355   return m_bRunning;
 356 }
 357
 358 void CMusicInfoScanner::Stop()
 359 {
 360   if (m_bCanInterrupt)
 361     m_musicDatabase.Interupt();
 362
 363   StopThread(false);
 364 }
 365
 366 static void OnDirectoryScanned(const CStdString& strDirectory)
 367 {
 368   CGUIMessage msg(GUI_MSG_DIRECTORY_SCANNED, 0, 0, 0);
 369   msg.SetStringParam(strDirectory);
 370   g_windowManager.SendThreadMessage(msg);
 371 }
 372
 373 static CStdString Prettify(const CStdString& strDirectory)
 374 {
 375   CURL url(strDirectory);
 376   CStdString strStrippedPath = url.GetWithoutUserDetails();
 377   CURL::Decode(strStrippedPath);
 378
 379   return strStrippedPath;
 380 }
 381
 382 bool CMusicInfoScanner::DoScan(const CStdString& strDirectory)
 383 {
 384   if (m_handle)
 385     m_handle->SetText(Prettify(strDirectory));
 386
 387   // Discard all excluded files defined by m_musicExcludeRegExps
 388   CStdStringArray regexps = g_advancedSettings.m_audioExcludeFromScanRegExps;
 389   if (CUtil::ExcludeFileOrFolder(strDirectory, regexps))
 390     return true;
 391
 392   // load subfolder
 393   CFileItemList items;
 394   CDirectory::GetDirectory(strDirectory, items, g_advancedSettings.m_musicExtensions + "|.jpg|.tbn|.lrc|.cdg");
 395
 396   // sort and get the path hash.  Note that we don't filter .cue sheet items here as we want
 397   // to detect changes in the .cue sheet as well.  The .cue sheet items only need filtering
 398   // if we have a changed hash.
 399   items.Sort(SortByLabel, SortOrderAscending);
 400   CStdString hash;
 401   GetPathHash(items, hash);
 402
 403   // check whether we need to rescan or not
 404   CStdString dbHash;
 405   if ((m_flags & SCAN_RESCAN) || !m_musicDatabase.GetPathHash(strDirectory, dbHash) || dbHash != hash)
 406   { // path has changed - rescan
 407     if (dbHash.IsEmpty())
 408       CLog::Log(LOGDEBUG, "%s Scanning dir '%s' as not in the database", __FUNCTION__, strDirectory.c_str());
 409     else
 410       CLog::Log(LOGDEBUG, "%s Rescanning dir '%s' due to change", __FUNCTION__, strDirectory.c_str());
 411
 412     // filter items in the sub dir (for .cue sheet support)
 413     items.FilterCueItems();
 414     items.Sort(SortByLabel, SortOrderAscending);
 415
 416     // and then scan in the new information
 417     if (RetrieveMusicInfo(strDirectory, items) > 0)
 418     {
 419       if (m_handle)
 420         OnDirectoryScanned(strDirectory);
 421     }
 422
 423     // save information about this folder
 424     m_musicDatabase.SetPathHash(strDirectory, hash);
 425   }
 426   else
 427   { // path is the same - no need to rescan
 428     CLog::Log(LOGDEBUG, "%s Skipping dir '%s' due to no change", __FUNCTION__, strDirectory.c_str());
 429     m_currentItem += CountFiles(items, false);  // false for non-recursive
 430
 431     // updated the dialog with our progress
 432     if (m_handle)
 433     {
 434       if (m_itemCount>0)
 435         m_handle->SetPercentage(m_currentItem/(float)m_itemCount*100);
 436       OnDirectoryScanned(strDirectory);
 437     }
 438   }
 439
 440   // now scan the subfolders
 441   for (int i = 0; i < items.Size(); ++i)
 442   {
 443     CFileItemPtr pItem = items[i];
 444
 445     if (m_bStop)
 446       break;
 447     // if we have a directory item (non-playlist) we then recurse into that folder
 448     if (pItem->m_bIsFolder && !pItem->IsParentFolder() && !pItem->IsPlayList())
 449     {
 450       CStdString strPath=pItem->GetPath();
 451       if (!DoScan(strPath))
 452       {
 453         m_bStop = true;
 454       }
 455     }
 456   }
 457
 458   return !m_bStop;
 459 }
 460
 461 INFO_RET CMusicInfoScanner::ScanTags(const CFileItemList& items, CFileItemList& scannedItems)
 462 {
 463   CStdStringArray regexps = g_advancedSettings.m_audioExcludeFromScanRegExps;
 464
 465   for (int i = 0; i < items.Size(); ++i)
 466   {
 467     if (m_bStop)
 468       return INFO_CANCELLED;
 469
 470     CFileItemPtr pItem = items[i];
 471
 472     if (CUtil::ExcludeFileOrFolder(pItem->GetPath(), regexps))
 473       continue;
 474
 475     if (pItem->m_bIsFolder || pItem->IsPlayList() || pItem->IsPicture() || pItem->IsLyrics())
 476       continue;
 477
 478     m_currentItem++;
 479
 480     CMusicInfoTag& tag = *pItem->GetMusicInfoTag();
 481     if (!tag.Loaded())
 482     {
 483       auto_ptr<IMusicInfoTagLoader> pLoader (CMusicInfoTagLoaderFactory::CreateLoader(pItem->GetPath()));
 484       if (NULL != pLoader.get())
 485         pLoader->Load(pItem->GetPath(), tag);
 486     }
 487
 488     if (m_handle && m_itemCount>0)
 489       m_handle->SetPercentage(m_currentItem/(float)m_itemCount*100);
 490
 491     if (!tag.Loaded())
 492     {
 493       CLog::Log(LOGDEBUG, "%s - No tag found for: %s", __FUNCTION__, pItem->GetPath().c_str());
 494       continue;
 495     }
 496     scannedItems.Add(pItem);
 497   }
 498   return INFO_ADDED;
 499 }
 500
 501 static bool SortSongsByTrack(const CSong& song, const CSong& song2)
 502 {
 503   return song.iTrack < song2.iTrack;
 504 }
 505
 506 void CMusicInfoScanner::FileItemsToAlbums(CFileItemList& items, VECALBUMS& albums, MAPSONGS* songsMap /* = NULL */)
 507 {
 508   /*
 509    * Step 1: Convert the FileItems into Songs.
 510    * If they're MB tagged, create albums directly from the FileItems.
 511    * If they're non-MB tagged, index them by album name ready for step 2.
 512    */
 513   map<string, VECSONGS> songsByAlbumNames;
 514   for (int i = 0; i < items.Size(); ++i)
 515   {
 516     CMusicInfoTag& tag = *items[i]->GetMusicInfoTag();
 517     CSong song(*items[i]);
 518
 519     // keep the db-only fields intact on rescan...
 520     if (songsMap != NULL)
 521     {
 522       MAPSONGS::iterator it = songsMap->find(items[i]->GetPath());
 523       if (it != songsMap->end())
 524       {
 525         song.iTimesPlayed = it->second.iTimesPlayed;
 526         song.lastPlayed = it->second.lastPlayed;
 527         song.iKaraokeNumber = it->second.iKaraokeNumber;
 528         if (song.rating == '0')    song.rating = it->second.rating;
 529         if (song.strThumb.empty()) song.strThumb = it->second.strThumb;
 530       }
 531     }
 532
 533     if (!tag.GetMusicBrainzAlbumID().empty())
 534     {
 535       VECALBUMS::iterator it;
 536       for (it = albums.begin(); it != albums.end(); ++it)
 537         if (it->strMusicBrainzAlbumID.Equals(tag.GetMusicBrainzAlbumID()))
 538           break;
 539
 540       if (it == albums.end())
 541       {
 542         CAlbum album(*items[i]);
 543         album.songs.push_back(song);
 544         albums.push_back(album);
 545       }
 546       else
 547         it->songs.push_back(song);
 548     }
 549     else
 550       songsByAlbumNames[tag.GetAlbum()].push_back(song);
 551   }
 552
 553   /*
 554    Step 2: Split into unique albums based on album name and album artist
 555    In the case where the album artist is unknown, we use the primary artist
 556    (i.e. first artist from each song).
 557    */
 558   for (map<string, VECSONGS>::iterator songsByAlbumName = songsByAlbumNames.begin(); songsByAlbumName != songsByAlbumNames.end(); ++songsByAlbumName)
 559   {
 560     VECSONGS &songs = songsByAlbumName->second;
 561     // sort the songs by tracknumber to identify duplicate track numbers
 562     sort(songs.begin(), songs.end(), SortSongsByTrack);
 563
 564     // map the songs to their primary artists
 565     bool tracksOverlap = false;
 566     bool hasAlbumArtist = false;
 567     bool isCompilation = true;
 568
 569     map<string, vector<CSong *> > artists;
 570     for (VECSONGS::iterator song = songs.begin(); song != songs.end(); ++song)
 571     {
 572       // test for song overlap
 573       if (song != songs.begin() && song->iTrack == (song - 1)->iTrack)
 574         tracksOverlap = true;
 575
 576       if (!song->bCompilation)
 577         isCompilation = false;
 578
 579       // get primary artist
 580       string primary;
 581       if (!song->albumArtist.empty())
 582       {
 583         primary = song->albumArtist[0];
 584         hasAlbumArtist = true;
 585       }
 586       else if (!song->artist.empty())
 587         primary = song->artist[0];
 588
 589       // add to the artist map
 590       artists[primary].push_back(&(*song));
 591     }
 592
 593     /*
 594      We have a compilation if
 595      1. album name is non-empty AND
 596      2a. no tracks overlap OR
 597      2b. all tracks are marked as part of compilation AND
 598      3a. a unique primary artist is specified as "various" or "various artists" OR
 599      3b. we have at least two primary artists and no album artist specified.
 600      */
 601     bool compilation = !songsByAlbumName->first.empty() && (isCompilation || !tracksOverlap); // 1+2b+2a
 602     if (artists.size() == 1)
 603     {
 604       string artist = artists.begin()->first; StringUtils::ToLower(artist);
 605       if (!StringUtils::EqualsNoCase(artist, "various") &&
 606           !StringUtils::EqualsNoCase(artist, "various artists")) // 3a
 607         compilation = false;
 608     }
 609     else if (hasAlbumArtist) // 3b
 610       compilation = false;
 611
 612     if (compilation)
 613     {
 614       CLog::Log(LOGDEBUG, "Album '%s' is a compilation as there's no overlapping tracks and %s", songsByAlbumName->first.c_str(), hasAlbumArtist ? "the album artist is 'Various'" : "there is more than one unique artist");
 615       artists.clear();
 616       std::string various = g_localizeStrings.Get(340); // Various Artists
 617       vector<string> va; va.push_back(various);
 618       for (VECSONGS::iterator song = songs.begin(); song != songs.end(); ++song)
 619       {
 620         song->albumArtist = va;
 621         artists[various].push_back(&(*song));
 622       }
 623     }
 624
 625     /*
 626      Step 3: Find the common albumartist for each song and assign
 627      albumartist to those tracks that don't have it set.
 628      */
 629     for (map<string, vector<CSong *> >::iterator j = artists.begin(); j != artists.end(); ++j)
 630     {
 631       // find the common artist for these songs
 632       vector<CSong *> &artistSongs = j->second;
 633       vector<string> common = artistSongs.front()->albumArtist.empty() ? artistSongs.front()->artist : artistSongs.front()->albumArtist;
 634       for (vector<CSong *>::iterator k = artistSongs.begin() + 1; k != artistSongs.end(); ++k)
 635       {
 636         unsigned int match = 0;
 637         vector<string> &compare = (*k)->albumArtist.empty() ? (*k)->artist : (*k)->albumArtist;
 638         for (; match < common.size() && match < compare.size(); match++)
 639         {
 640           if (compare[match] != common[match])
 641             break;
 642         }
 643         common.erase(common.begin() + match, common.end());
 644       }
 645
 646       /*
 647        Step 4: Assign the album artist for each song that doesn't have it set
 648        and add to the album vector
 649        */
 650       CAlbum album;
 651       album.strAlbum = songsByAlbumName->first;
 652       album.artist = common;
 653       for (vector<string>::iterator it = common.begin(); it != common.end(); ++it)
 654       {
 655         CStdString strJoinPhrase = (it == --common.end() ? "" : g_advancedSettings.m_musicItemSeparator);
 656         CArtistCredit artistCredit(*it, strJoinPhrase);
 657         album.artistCredits.push_back(artistCredit);
 658       }
 659       album.bCompilation = compilation;
 660       for (vector<CSong *>::iterator k = artistSongs.begin(); k != artistSongs.end(); ++k)
 661       {
 662         if ((*k)->albumArtist.empty())
 663           (*k)->albumArtist = common;
 664         // TODO: in future we may wish to union up the genres, for now we assume they're the same
 665         album.genre = (*k)->genre;
 666         //       in addition, we may want to use year as discriminating for albums
 667         album.iYear = (*k)->iYear;
 668         album.songs.push_back(**k);
 669       }
 670       albums.push_back(album);
 671     }
 672   }
 673 }
 674
 675 int CMusicInfoScanner::RetrieveMusicInfo(const CStdString& strDirectory, CFileItemList& items)
 676 {
 677   MAPSONGS songsMap;
 678
 679   // get all information for all files in current directory from database, and remove them
 680   if (m_musicDatabase.RemoveSongsFromPath(strDirectory, songsMap))
 681     m_needsCleanup = true;
 682
 683   CFileItemList scannedItems;
 684   if (ScanTags(items, scannedItems) == INFO_CANCELLED)
 685     return 0;
 686
 687   VECALBUMS albums;
 688   FileItemsToAlbums(scannedItems, albums, &songsMap);
 689   FindArtForAlbums(albums, items.GetPath());
 690
 691   int numAdded = 0;
 692   ADDON::AddonPtr addon;
 693   ADDON::ScraperPtr albumScraper;
 694   ADDON::ScraperPtr artistScraper;
 695   if(ADDON::CAddonMgr::Get().GetDefault(ADDON::ADDON_SCRAPER_ALBUMS, addon))
 696     albumScraper = boost::dynamic_pointer_cast<ADDON::CScraper>(addon);
 697
 698   if(ADDON::CAddonMgr::Get().GetDefault(ADDON::ADDON_SCRAPER_ARTISTS, addon))
 699     artistScraper = boost::dynamic_pointer_cast<ADDON::CScraper>(addon);
 700
 701   // Add each album
 702   for (VECALBUMS::iterator album = albums.begin(); album != albums.end(); ++album)
 703   {
 704     if (m_bStop)
 705       break;
 706
 707     album->strPath = strDirectory;
 708     m_musicDatabase.BeginTransaction();
 709
 710     // Check if the album has already been downloaded or failed
 711     map<CAlbum, CAlbum>::iterator cachedAlbum = m_albumCache.find(*album);
 712     if (cachedAlbum == m_albumCache.end())
 713     {
 714       // No - download the information
 715       CMusicAlbumInfo albumInfo;
 716       INFO_RET albumDownloadStatus = INFO_NOT_FOUND;
 717       if ((m_flags & SCAN_ONLINE) && albumScraper)
 718         albumDownloadStatus = DownloadAlbumInfo(*album, albumScraper, albumInfo);
 719
 720       if (albumDownloadStatus == INFO_ADDED || albumDownloadStatus == INFO_HAVE_ALREADY)
 721       {
 722         CAlbum &downloadedAlbum = albumInfo.GetAlbum();
 723         downloadedAlbum.idAlbum = m_musicDatabase.AddAlbum(downloadedAlbum.strAlbum,
 724                                                            downloadedAlbum.strMusicBrainzAlbumID,
 725                                                            downloadedAlbum.GetArtistString(),
 726                                                            downloadedAlbum.GetGenreString(),
 727                                                            downloadedAlbum.iYear,
 728                                                            downloadedAlbum.bCompilation);
 729         m_musicDatabase.SetAlbumInfo(downloadedAlbum.idAlbum,
 730                                      downloadedAlbum,
 731                                      downloadedAlbum.songs);
 732         m_musicDatabase.SetArtForItem(downloadedAlbum.idAlbum,
 733                                       "album", album->art);
 734         GetAlbumArtwork(downloadedAlbum.idAlbum, downloadedAlbum);
 735         m_albumCache.insert(make_pair(*album, albumInfo.GetAlbum()));
 736       }
 737       else if (albumDownloadStatus == INFO_CANCELLED)
 738         break;
 739       else
 740       {
 741         // No download info, fallback to already gathered (eg. local) information/art (if any)
 742         album->idAlbum = m_musicDatabase.AddAlbum(album->strAlbum,
 743                                                   album->strMusicBrainzAlbumID,
 744                                                   album->GetArtistString(),
 745                                                   album->GetGenreString(),
 746                                                   album->iYear,
 747                                                   album->bCompilation);
 748         if (!album->art.empty())
 749           m_musicDatabase.SetArtForItem(album->idAlbum,
 750                                         "album", album->art);
 751         m_albumCache.insert(make_pair(*album, *album));
 752       }
 753
 754       // Update the cache pointer with our newly created info
 755       cachedAlbum = m_albumCache.find(*album);
 756     }
 757
 758     if (m_bStop)
 759       break;
 760
 761     // Add the album artists
 762     for (VECARTISTCREDITS::iterator artistCredit = cachedAlbum->second.artistCredits.begin(); artistCredit != cachedAlbum->second.artistCredits.end(); ++artistCredit)
 763     {
 764       if (m_bStop)
 765         break;
 766
 767       // Check if the artist has already been downloaded or failed
 768       map<CArtistCredit, CArtist>::iterator cachedArtist = m_artistCache.find(*artistCredit);
 769       if (cachedArtist == m_artistCache.end())
 770       {
 771         CArtist artistTmp;
 772         artistTmp.strArtist = artistCredit->GetArtist();
 773         artistTmp.strMusicBrainzArtistID = artistCredit->GetMusicBrainzArtistID();
 774         URIUtils::GetParentPath(album->strPath, artistTmp.strPath);
 775
 776         // No - download the information
 777         CMusicArtistInfo artistInfo;
 778         INFO_RET artistDownloadStatus = INFO_NOT_FOUND;
 779         if ((m_flags & SCAN_ONLINE) && artistScraper)
 780           artistDownloadStatus = DownloadArtistInfo(artistTmp, artistScraper, artistInfo);
 781
 782         if (artistDownloadStatus == INFO_ADDED || artistDownloadStatus == INFO_HAVE_ALREADY)
 783         {
 784           CArtist &downloadedArtist = artistInfo.GetArtist();
 785           downloadedArtist.idArtist = m_musicDatabase.AddArtist(downloadedArtist.strArtist,
 786                                                                 downloadedArtist.strMusicBrainzArtistID);
 787           m_musicDatabase.SetArtistInfo(downloadedArtist.idArtist,
 788                                         downloadedArtist);
 789
 790           URIUtils::GetParentPath(album->strPath, downloadedArtist.strPath);
 791           map<string, string> artwork = GetArtistArtwork(downloadedArtist);
 792           // check thumb stuff
 793           m_musicDatabase.SetArtForItem(downloadedArtist.idArtist, "artist", artwork);
 794           m_artistCache.insert(make_pair(*artistCredit, downloadedArtist));
 795         }
 796         else if (artistDownloadStatus == INFO_CANCELLED)
 797           break;
 798         else
 799         {
 800           // Cache the lookup failure so we don't retry
 801           artistTmp.idArtist = m_musicDatabase.AddArtist(artistCredit->GetArtist(), artistCredit->GetMusicBrainzArtistID());
 802           m_artistCache.insert(make_pair(*artistCredit, artistTmp));
 803         }
 804
 805         // Update the cache pointer with our newly created info
 806         cachedArtist = m_artistCache.find(*artistCredit);
 807       }
 808
 809       m_musicDatabase.AddAlbumArtist(cachedArtist->second.idArtist,
 810                                      cachedAlbum->second.idAlbum,
 811                                      artistCredit->GetJoinPhrase(),
 812                                      artistCredit == cachedAlbum->second.artistCredits.begin() ? false : true,
 813                                      std::distance(cachedAlbum->second.artistCredits.begin(), artistCredit));
 814     }
 815
 816     if (m_bStop)
 817       break;
 818
 819     for (VECSONGS::iterator song = album->songs.begin(); song != album->songs.end(); ++song)
 820     {
 821       song->idAlbum = cachedAlbum->second.idAlbum;
 822       song->idSong = m_musicDatabase.AddSong(cachedAlbum->second.idAlbum,
 823                                              song->strTitle, song->strMusicBrainzTrackID,
 824                                              song->strFileName, song->strComment,
 825                                              song->strThumb,
 826                                              song->artist, song->genre,
 827                                              song->iTrack, song->iDuration, song->iYear,
 828                                              song->iTimesPlayed, song->iStartOffset,
 829                                              song->iEndOffset,
 830                                              song->lastPlayed,
 831                                              song->rating,
 832                                              song->iKaraokeNumber);
 833       for (VECARTISTCREDITS::iterator artistCredit = song->artistCredits.begin(); artistCredit != song->artistCredits.end(); ++artistCredit)
 834       {
 835         if (m_bStop)
 836           break;
 837
 838         // Check if the artist has already been downloaded or failed
 839         map<CArtistCredit, CArtist>::iterator cachedArtist = m_artistCache.find(*artistCredit);
 840         if (cachedArtist == m_artistCache.end())
 841         {
 842           CArtist artistTmp;
 843           artistTmp.strArtist = artistCredit->GetArtist();
 844           artistTmp.strMusicBrainzArtistID = artistCredit->GetMusicBrainzArtistID();
 845           URIUtils::GetParentPath(album->strPath, artistTmp.strPath); // FIXME
 846
 847           // No - download the information
 848           CMusicArtistInfo artistInfo;
 849           INFO_RET artistDownloadStatus = INFO_NOT_FOUND;
 850           if ((m_flags & SCAN_ONLINE) && artistScraper)
 851             artistDownloadStatus = DownloadArtistInfo(artistTmp, artistScraper, artistInfo);
 852
 853           if (artistDownloadStatus == INFO_ADDED || artistDownloadStatus == INFO_HAVE_ALREADY)
 854           {
 855             CArtist &downloadedArtist = artistInfo.GetArtist();
 856             downloadedArtist.idArtist = m_musicDatabase.AddArtist(downloadedArtist.strArtist,
 857                                                                   downloadedArtist.strMusicBrainzArtistID);
 858             m_musicDatabase.SetArtistInfo(downloadedArtist.idArtist,
 859                                           downloadedArtist);
 860             // check thumb stuff
 861             URIUtils::GetParentPath(album->strPath, downloadedArtist.strPath);
 862             map<string, string> artwork = GetArtistArtwork(downloadedArtist);
 863             m_musicDatabase.SetArtForItem(downloadedArtist.idArtist, "artist", artwork);
 864             m_artistCache.insert(make_pair(*artistCredit, downloadedArtist));
 865           }
 866           else if (artistDownloadStatus == INFO_CANCELLED)
 867             break;
 868           else
 869           {
 870             // Cache the lookup failure so we don't retry
 871             artistTmp.idArtist = m_musicDatabase.AddArtist(artistCredit->GetArtist(), artistCredit->GetMusicBrainzArtistID());
 872             m_artistCache.insert(make_pair(*artistCredit, artistTmp));
 873           }
 874
 875           // Update the cache pointer with our newly created info
 876           cachedArtist = m_artistCache.find(*artistCredit);
 877         }
 878
 879         m_musicDatabase.AddSongArtist(cachedArtist->second.idArtist,
 880                                       song->idSong,
 881                                       g_advancedSettings.m_musicItemSeparator, // we don't have song artist breakdowns from scrapers, yet
 882                                       artistCredit == song->artistCredits.begin() ? false : true,
 883                                       std::distance(song->artistCredits.begin(), artistCredit));
 884       }
 885     }
 886
 887     if (m_bStop)
 888       break;
 889
 890     // Commit the album to the DB
 891     m_musicDatabase.CommitTransaction();
 892     numAdded += album->songs.size();
 893   }
 894
 895   if (m_bStop)
 896     m_musicDatabase.RollbackTransaction();
 897
 898   if (m_handle)
 899     m_handle->SetTitle(g_localizeStrings.Get(505));
 900
 901   return numAdded;
 902 }
 903
 904 void CMusicInfoScanner::FindArtForAlbums(VECALBUMS &albums, const CStdString &path)
 905 {
 906   /*
 907    If there's a single album in the folder, then art can be taken from
 908    the folder art.
 909    */
 910   std::string albumArt;
 911   if (albums.size() == 1)
 912   {
 913     CFileItem album(path, true);
 914     albumArt = album.GetUserMusicThumb(true);
 915     if (!albumArt.empty())
 916       albums[0].art["thumb"] = albumArt;
 917   }
 918   for (VECALBUMS::iterator i = albums.begin(); i != albums.end(); ++i)
 919   {
 920     CAlbum &album = *i;
 921
 922     if (albums.size() != 1)
 923       albumArt = "";
 924
 925     /*
 926      Find art that is common across these items
 927      If we find a single art image we treat it as the album art
 928      and discard song art else we use first as album art and
 929      keep everything as song art.
 930      */
 931     bool singleArt = true;
 932     CSong *art = NULL;
 933     for (VECSONGS::iterator k = album.songs.begin(); k != album.songs.end(); ++k)
 934     {
 935       CSong &song = *k;
 936       if (song.HasArt())
 937       {
 938         if (art && !art->ArtMatches(song))
 939         {
 940           singleArt = false;
 941           break;
 942         }
 943         if (!art)
 944           art = &song;
 945       }
 946     }
 947
 948     /*
 949       assign the first art found to the album - better than no art at all
 950     */
 951
 952     if (art && albumArt.empty())
 953     {
 954       if (!art->strThumb.empty())
 955         albumArt = art->strThumb;
 956       else
 957         albumArt = CTextureUtils::GetWrappedImageURL(art->strFileName, "music");
 958     }
 959
 960     if (!albumArt.empty())
 961       album.art["thumb"] = albumArt;
 962
 963     if (singleArt)
 964     { //if singleArt then we can clear the artwork for all songs
 965       for (VECSONGS::iterator k = album.songs.begin(); k != album.songs.end(); ++k)
 966         k->strThumb.clear();
 967     }
 968     else
 969     { // more than one piece of art was found for these songs, so cache per song
 970       for (VECSONGS::iterator k = album.songs.begin(); k != album.songs.end(); ++k)
 971       {
 972         if (k->strThumb.empty() && !k->embeddedArt.empty())
 973           k->strThumb = CTextureUtils::GetWrappedImageURL(k->strFileName, "music");
 974       }
 975     }
 976   }
 977   if (albums.size() == 1 && !albumArt.empty())
 978   {
 979     // assign to folder thumb as well
 980     CFileItem albumItem(path, true);
 981     CMusicThumbLoader loader;
 982     loader.SetCachedImage(albumItem, "thumb", albumArt);
 983   }
 984 }
 985
 986 int CMusicInfoScanner::GetPathHash(const CFileItemList &items, CStdString &hash)
 987 {
 988   // Create a hash based on the filenames, filesize and filedate.  Also count the number of files
 989   if (0 == items.Size()) return 0;
 990   XBMC::XBMC_MD5 md5state;
 991   int count = 0;
 992   for (int i = 0; i < items.Size(); ++i)
 993   {
 994     const CFileItemPtr pItem = items[i];
 995     md5state.append(pItem->GetPath());
 996     md5state.append((unsigned char *)&pItem->m_dwSize, sizeof(pItem->m_dwSize));
 997     FILETIME time = pItem->m_dateTime;
 998     md5state.append((unsigned char *)&time, sizeof(FILETIME));
 999     if (pItem->IsAudio() && !pItem->IsPlayList() && !pItem->IsNFO())
1000       count++;
1001   }
1002   md5state.getDigest(hash);
1003   return count;
1004 }
1005
1006 INFO_RET CMusicInfoScanner::UpdateDatabaseAlbumInfo(const CStdString& strPath, CMusicAlbumInfo& albumInfo, bool bAllowSelection, CGUIDialogProgress* pDialog /* = NULL */)
1007 {
1008   m_musicDatabase.Open();
1009   CQueryParams params;
1010   CDirectoryNode::GetDatabaseInfo(strPath, params);
1011
1012   if (params.GetAlbumId() == -1)
1013     return INFO_ERROR;
1014
1015   CAlbum album;
1016   m_musicDatabase.GetAlbumInfo(params.GetAlbumId(), album, &album.songs);
1017
1018   // find album info
1019   ADDON::ScraperPtr scraper;
1020   bool result = m_musicDatabase.GetScraperForPath(strPath, scraper, ADDON::ADDON_SCRAPER_ALBUMS);
1021
1022   m_musicDatabase.Close();
1023
1024   if (!result || !scraper)
1025     return INFO_ERROR;
1026
1027 loop:
1028   CLog::Log(LOGDEBUG, "%s downloading info for: %s", __FUNCTION__, album.strAlbum.c_str());
1029   INFO_RET albumDownloadStatus = DownloadAlbumInfo(album, scraper, albumInfo, pDialog);
1030   if (albumDownloadStatus == INFO_NOT_FOUND)
1031   {
1032     if (pDialog && bAllowSelection)
1033     {
1034       if (!CGUIKeyboardFactory::ShowAndGetInput(album.strAlbum, g_localizeStrings.Get(16011), false))
1035         return INFO_CANCELLED;
1036
1037       CStdString strTempArtist(StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
1038       if (!CGUIKeyboardFactory::ShowAndGetInput(strTempArtist, g_localizeStrings.Get(16025), false))
1039         return INFO_CANCELLED;
1040
1041       album.artist = StringUtils::Split(strTempArtist, g_advancedSettings.m_musicItemSeparator);
1042       goto loop;
1043     }
1044   }
1045   else if (albumDownloadStatus == INFO_ADDED)
1046   {
1047     m_musicDatabase.Open();
1048     m_musicDatabase.SetAlbumInfo(params.GetAlbumId(), albumInfo.GetAlbum(), albumInfo.GetAlbum().songs);
1049     GetAlbumArtwork(params.GetAlbumId(), albumInfo.GetAlbum());
1050     albumInfo.SetLoaded(true);
1051     m_musicDatabase.Close();
1052   }
1053   return albumDownloadStatus;
1054 }
1055
1056 INFO_RET CMusicInfoScanner::UpdateDatabaseArtistInfo(const CStdString& strPath, CMusicArtistInfo& artistInfo, bool bAllowSelection, CGUIDialogProgress* pDialog /* = NULL */)
1057 {
1058   m_musicDatabase.Open();
1059   CQueryParams params;
1060   CDirectoryNode::GetDatabaseInfo(strPath, params);
1061
1062   if (params.GetArtistId() == -1)
1063     return INFO_ERROR;
1064
1065   CArtist artist;
1066   m_musicDatabase.GetArtistInfo(params.GetArtistId(), artist);
1067
1068   // find album info
1069   ADDON::ScraperPtr scraper;
1070   if (!m_musicDatabase.GetScraperForPath(strPath, scraper, ADDON::ADDON_SCRAPER_ARTISTS) || !scraper)
1071     return INFO_ERROR;
1072   m_musicDatabase.Close();
1073
1074 loop:
1075   CLog::Log(LOGDEBUG, "%s downloading info for: %s", __FUNCTION__, artist.strArtist.c_str());
1076   INFO_RET artistDownloadStatus = DownloadArtistInfo(artist, scraper, artistInfo, pDialog);
1077   if (artistDownloadStatus == INFO_NOT_FOUND)
1078   {
1079     if (pDialog && bAllowSelection)
1080     {
1081       if (!CGUIKeyboardFactory::ShowAndGetInput(artist.strArtist, g_localizeStrings.Get(16025), false))
1082         return INFO_CANCELLED;
1083       goto loop;
1084     }
1085   }
1086   else if (artistDownloadStatus == INFO_ADDED)
1087   {
1088     m_musicDatabase.Open();
1089     m_musicDatabase.SetArtistInfo(params.GetArtistId(), artistInfo.GetArtist());
1090     m_musicDatabase.GetArtistPath(params.GetArtistId(), artist.strPath);
1091     map<string, string> artwork = GetArtistArtwork(artist);
1092     m_musicDatabase.SetArtForItem(params.GetArtistId(), "artist", artwork);
1093     artistInfo.SetLoaded();
1094     m_musicDatabase.Close();
1095   }
1096   return artistDownloadStatus;
1097 }
1098
1099 #define THRESHOLD .95f
1100
1101 INFO_RET CMusicInfoScanner::DownloadAlbumInfo(const CAlbum& album, ADDON::ScraperPtr& info, CMusicAlbumInfo& albumInfo, CGUIDialogProgress* pDialog)
1102 {
1103   if (m_handle)
1104   {
1105     m_handle->SetTitle(StringUtils::Format(g_localizeStrings.Get(20321), info->Name().c_str()));
1106     m_handle->SetText(StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator) + " - " + album.strAlbum);
1107   }
1108
1109   // clear our scraper cache
1110   info->ClearCache();
1111
1112   CMusicInfoScraper scraper(info);
1113   bool bMusicBrainz = false;
1114   if (!album.strMusicBrainzAlbumID.empty())
1115   {
1116     CScraperUrl musicBrainzURL;
1117     if (ResolveMusicBrainz(album.strMusicBrainzAlbumID, info, scraper, musicBrainzURL))
1118     {
1119       CMusicAlbumInfo albumNfo("nfo", musicBrainzURL);
1120       scraper.GetAlbums().clear();
1121       scraper.GetAlbums().push_back(albumNfo);
1122       bMusicBrainz = true;
1123     }
1124   }
1125
1126   // handle nfo files
1127   CStdString strNfo = URIUtils::AddFileToFolder(album.strPath, "album.nfo");
1128   CNfoFile::NFOResult result = CNfoFile::NO_NFO;
1129   CNfoFile nfoReader;
1130   if (XFILE::CFile::Exists(strNfo))
1131   {
1132     CLog::Log(LOGDEBUG,"Found matching nfo file: %s", strNfo.c_str());
1133     result = nfoReader.Create(strNfo, info, -1, album.strPath);
1134     if (result == CNfoFile::FULL_NFO)
1135     {
1136       CLog::Log(LOGDEBUG, "%s Got details from nfo", __FUNCTION__);
1137       nfoReader.GetDetails(albumInfo.GetAlbum());
1138       return INFO_ADDED;
1139     }
1140     else if (result == CNfoFile::URL_NFO || result == CNfoFile::COMBINED_NFO)
1141     {
1142       CScraperUrl scrUrl(nfoReader.ScraperUrl());
1143       CMusicAlbumInfo albumNfo("nfo",scrUrl);
1144       info = nfoReader.GetScraperInfo();
1145       CLog::Log(LOGDEBUG,"-- nfo-scraper: %s",info->Name().c_str());
1146       CLog::Log(LOGDEBUG,"-- nfo url: %s", scrUrl.m_url[0].m_url.c_str());
1147       scraper.SetScraperInfo(info);
1148       scraper.GetAlbums().clear();
1149       scraper.GetAlbums().push_back(albumNfo);
1150     }
1151     else
1152       CLog::Log(LOGERROR,"Unable to find an url in nfo file: %s", strNfo.c_str());
1153   }
1154
1155   if (!scraper.CheckValidOrFallback(CSettings::Get().GetString("musiclibrary.albumsscraper")))
1156   { // the current scraper is invalid, as is the default - bail
1157     CLog::Log(LOGERROR, "%s - current and default scrapers are invalid.  Pick another one", __FUNCTION__);
1158     return INFO_ERROR;
1159   }
1160
1161   if (!scraper.GetAlbumCount())
1162   {
1163     scraper.FindAlbumInfo(album.strAlbum, StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
1164
1165     while (!scraper.Completed())
1166     {
1167       if (m_bStop)
1168       {
1169         scraper.Cancel();
1170         return INFO_CANCELLED;
1171       }
1172       Sleep(1);
1173     }
1174   }
1175
1176   CGUIDialogSelect *pDlg = NULL;
1177   int iSelectedAlbum=0;
1178   if (result == CNfoFile::NO_NFO && !bMusicBrainz)
1179   {
1180     iSelectedAlbum = -1; // set negative so that we can detect a failure
1181     if (scraper.Succeeded() && scraper.GetAlbumCount() >= 1)
1182     {
1183       double bestRelevance = 0;
1184       double minRelevance = THRESHOLD;
1185       if (scraper.GetAlbumCount() > 1) // score the matches
1186       {
1187         //show dialog with all albums found
1188         if (pDialog)
1189         {
1190           pDlg = (CGUIDialogSelect*)g_windowManager.GetWindow(WINDOW_DIALOG_SELECT);
1191           pDlg->SetHeading(g_localizeStrings.Get(181).c_str());
1192           pDlg->Reset();
1193           pDlg->EnableButton(true, 413); // manual
1194         }
1195
1196         for (int i = 0; i < scraper.GetAlbumCount(); ++i)
1197         {
1198           CMusicAlbumInfo& info = scraper.GetAlbum(i);
1199           double relevance = info.GetRelevance();
1200           if (relevance < 0)
1201             relevance = CUtil::AlbumRelevance(info.GetAlbum().strAlbum, album.strAlbum, StringUtils::Join(info.GetAlbum().artist, g_advancedSettings.m_musicItemSeparator), StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
1202
1203           // if we're doing auto-selection (ie querying all albums at once, then allow 95->100% for perfect matches)
1204           // otherwise, perfect matches only
1205           if (relevance >= max(minRelevance, bestRelevance))
1206           { // we auto-select the best of these
1207             bestRelevance = relevance;
1208             iSelectedAlbum = i;
1209           }
1210           if (pDialog)
1211           {
1212             // set the label to [relevance]  album - artist
1213             CStdString strTemp = StringUtils::Format("[%0.2f]  %s", relevance, info.GetTitle2().c_str());
1214             CFileItem item(strTemp);
1215             item.m_idepth = i; // use this to hold the index of the album in the scraper
1216             pDlg->Add(&item);
1217           }
1218           if (relevance > .99f) // we're so close, no reason to search further
1219             break;
1220         }
1221
1222         if (pDialog && bestRelevance < THRESHOLD)
1223         {
1224           pDlg->Sort(false);
1225           pDlg->DoModal();
1226
1227           // and wait till user selects one
1228           if (pDlg->GetSelectedLabel() < 0)
1229           { // none chosen
1230             if (!pDlg->IsButtonPressed())
1231               return INFO_CANCELLED;
1232
1233             // manual button pressed
1234             CStdString strNewAlbum = album.strAlbum;
1235             if (!CGUIKeyboardFactory::ShowAndGetInput(strNewAlbum, g_localizeStrings.Get(16011), false)) return INFO_CANCELLED;
1236             if (strNewAlbum == "") return INFO_CANCELLED;
1237
1238             CStdString strNewArtist = StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator);
1239             if (!CGUIKeyboardFactory::ShowAndGetInput(strNewArtist, g_localizeStrings.Get(16025), false)) return INFO_CANCELLED;
1240
1241             pDialog->SetLine(0, strNewAlbum);
1242             pDialog->SetLine(1, strNewArtist);
1243             pDialog->Progress();
1244
1245             CAlbum newAlbum = album;
1246             newAlbum.strAlbum = strNewAlbum;
1247             newAlbum.artist = StringUtils::Split(strNewArtist, g_advancedSettings.m_musicItemSeparator);
1248
1249             return DownloadAlbumInfo(newAlbum, info, albumInfo, pDialog);
1250           }
1251           iSelectedAlbum = pDlg->GetSelectedItem()->m_idepth;
1252         }
1253       }
1254       else
1255       {
1256         CMusicAlbumInfo& info = scraper.GetAlbum(0);
1257         double relevance = info.GetRelevance();
1258         if (relevance < 0)
1259           relevance = CUtil::AlbumRelevance(info.GetAlbum().strAlbum,
1260                                             album.strAlbum,
1261                                             StringUtils::Join(info.GetAlbum().artist, g_advancedSettings.m_musicItemSeparator),
1262                                             StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
1263         if (relevance < THRESHOLD)
1264           return INFO_NOT_FOUND;
1265
1266         iSelectedAlbum = 0;
1267       }
1268     }
1269
1270     if (iSelectedAlbum < 0)
1271       return INFO_NOT_FOUND;
1272
1273   }
1274
1275   scraper.LoadAlbumInfo(iSelectedAlbum);
1276   while (!scraper.Completed())
1277   {
1278     if (m_bStop)
1279     {
1280       scraper.Cancel();
1281       return INFO_CANCELLED;
1282     }
1283     Sleep(1);
1284   }
1285
1286   if (!scraper.Succeeded())
1287     return INFO_ERROR;
1288
1289   albumInfo = scraper.GetAlbum(iSelectedAlbum);
1290
1291   if (result == CNfoFile::COMBINED_NFO)
1292     nfoReader.GetDetails(albumInfo.GetAlbum(), NULL, true);
1293
1294   return INFO_ADDED;
1295 }
1296
1297 void CMusicInfoScanner::GetAlbumArtwork(long id, const CAlbum &album)
1298 {
1299   if (album.thumbURL.m_url.size())
1300   {
1301     if (m_musicDatabase.GetArtForItem(id, "album", "thumb").empty())
1302     {
1303       string thumb = CScraperUrl::GetThumbURL(album.thumbURL.GetFirstThumb());
1304       if (!thumb.empty())
1305       {
1306         CTextureCache::Get().BackgroundCacheImage(thumb);
1307         m_musicDatabase.SetArtForItem(id, "album", "thumb", thumb);
1308       }
1309     }
1310   }
1311 }
1312
1313 INFO_RET CMusicInfoScanner::DownloadArtistInfo(const CArtist& artist, ADDON::ScraperPtr& info, MUSIC_GRABBER::CMusicArtistInfo& artistInfo, CGUIDialogProgress* pDialog)
1314 {
1315   if (m_handle)
1316   {
1317     m_handle->SetTitle(StringUtils::Format(g_localizeStrings.Get(20320), info->Name().c_str()));
1318     m_handle->SetText(artist.strArtist);
1319   }
1320
1321   // clear our scraper cache
1322   info->ClearCache();
1323
1324   CMusicInfoScraper scraper(info);
1325   bool bMusicBrainz = false;
1326   if (!artist.strMusicBrainzArtistID.empty())
1327   {
1328     CScraperUrl musicBrainzURL;
1329     if (ResolveMusicBrainz(artist.strMusicBrainzArtistID, info, scraper, musicBrainzURL))
1330     {
1331       CMusicArtistInfo artistNfo("nfo", musicBrainzURL);
1332       scraper.GetArtists().clear();
1333       scraper.GetArtists().push_back(artistNfo);
1334       bMusicBrainz = true;
1335     }
1336   }
1337
1338   // handle nfo files
1339   CStdString strNfo = URIUtils::AddFileToFolder(artist.strPath, "artist.nfo");
1340   CNfoFile::NFOResult result=CNfoFile::NO_NFO;
1341   CNfoFile nfoReader;
1342   if (XFILE::CFile::Exists(strNfo))
1343   {
1344     CLog::Log(LOGDEBUG,"Found matching nfo file: %s", strNfo.c_str());
1345     result = nfoReader.Create(strNfo, info);
1346     if (result == CNfoFile::FULL_NFO)
1347     {
1348       CLog::Log(LOGDEBUG, "%s Got details from nfo", __FUNCTION__);
1349       nfoReader.GetDetails(artistInfo.GetArtist());
1350       return INFO_ADDED;
1351     }
1352     else if (result == CNfoFile::URL_NFO || result == CNfoFile::COMBINED_NFO)
1353     {
1354       CScraperUrl scrUrl(nfoReader.ScraperUrl());
1355       CMusicArtistInfo artistNfo("nfo",scrUrl);
1356       info = nfoReader.GetScraperInfo();
1357       CLog::Log(LOGDEBUG,"-- nfo-scraper: %s",info->Name().c_str());
1358       CLog::Log(LOGDEBUG,"-- nfo url: %s", scrUrl.m_url[0].m_url.c_str());
1359       scraper.SetScraperInfo(info);
1360       scraper.GetArtists().push_back(artistNfo);
1361     }
1362     else
1363       CLog::Log(LOGERROR,"Unable to find an url in nfo file: %s", strNfo.c_str());
1364   }
1365
1366   if (!scraper.GetArtistCount())
1367   {
1368     scraper.FindArtistInfo(artist.strArtist);
1369
1370     while (!scraper.Completed())
1371     {
1372       if (m_bStop)
1373       {
1374         scraper.Cancel();
1375         return INFO_CANCELLED;
1376       }
1377       Sleep(1);
1378     }
1379   }
1380
1381   int iSelectedArtist = 0;
1382   if (result == CNfoFile::NO_NFO && !bMusicBrainz)
1383   {
1384     if (scraper.GetArtistCount() >= 1)
1385     {
1386       // now load the first match
1387       if (pDialog && scraper.GetArtistCount() > 1)
1388       {
1389         // if we found more then 1 album, let user choose one
1390         CGUIDialogSelect *pDlg = (CGUIDialogSelect*)g_windowManager.GetWindow(WINDOW_DIALOG_SELECT);
1391         if (pDlg)
1392         {
1393           pDlg->SetHeading(g_localizeStrings.Get(21890));
1394           pDlg->Reset();
1395           pDlg->EnableButton(true, 413); // manual
1396
1397           for (int i = 0; i < scraper.GetArtistCount(); ++i)
1398           {
1399             // set the label to artist
1400             CFileItem item(scraper.GetArtist(i).GetArtist());
1401             CStdString strTemp=scraper.GetArtist(i).GetArtist().strArtist;
1402             if (!scraper.GetArtist(i).GetArtist().strBorn.IsEmpty())
1403               strTemp += " ("+scraper.GetArtist(i).GetArtist().strBorn+")";
1404             if (!scraper.GetArtist(i).GetArtist().genre.empty())
1405             {
1406               CStdString genres = StringUtils::Join(scraper.GetArtist(i).GetArtist().genre, g_advancedSettings.m_musicItemSeparator);
1407               if (!genres.empty())
1408                 strTemp = StringUtils::Format("[%s] %s", genres.c_str(), strTemp.c_str());
1409             }
1410             item.SetLabel(strTemp);
1411             item.m_idepth = i; // use this to hold the index of the album in the scraper
1412             pDlg->Add(&item);
1413           }
1414           pDlg->DoModal();
1415
1416           // and wait till user selects one
1417           if (pDlg->GetSelectedLabel() < 0)
1418           { // none chosen
1419             if (!pDlg->IsButtonPressed())
1420               return INFO_CANCELLED;
1421
1422             // manual button pressed
1423             CStdString strNewArtist = artist.strArtist;
1424             if (!CGUIKeyboardFactory::ShowAndGetInput(strNewArtist, g_localizeStrings.Get(16025), false)) return INFO_CANCELLED;
1425
1426             if (pDialog)
1427             {
1428               pDialog->SetLine(0, strNewArtist);
1429               pDialog->Progress();
1430             }
1431
1432             CArtist newArtist;
1433             newArtist.strArtist = strNewArtist;
1434             return DownloadArtistInfo(newArtist, info, artistInfo, pDialog);
1435           }
1436           iSelectedArtist = pDlg->GetSelectedItem()->m_idepth;
1437         }
1438       }
1439     }
1440     else
1441       return INFO_NOT_FOUND;
1442   }
1443
1444   scraper.LoadArtistInfo(iSelectedArtist, artist.strArtist);
1445   while (!scraper.Completed())
1446   {
1447     if (m_bStop)
1448     {
1449       scraper.Cancel();
1450       return INFO_CANCELLED;
1451     }
1452     Sleep(1);
1453   }
1454
1455   if (!scraper.Succeeded())
1456     return INFO_ERROR;
1457
1458   artistInfo = scraper.GetArtist(iSelectedArtist);
1459
1460   if (result == CNfoFile::COMBINED_NFO)
1461     nfoReader.GetDetails(artistInfo.GetArtist(), NULL, true);
1462
1463   return INFO_ADDED;
1464 }
1465
1466 bool CMusicInfoScanner::ResolveMusicBrainz(const CStdString strMusicBrainzID, ScraperPtr &preferredScraper, CMusicInfoScraper &musicInfoScraper, CScraperUrl &musicBrainzURL)
1467 {
1468   // We have a MusicBrainz ID
1469   // Get a scraper that can resolve it to a MusicBrainz URL & force our
1470   // search directly to the specific album.
1471   bool bMusicBrainz = false;
1472   ADDON::TYPE type = ScraperTypeFromContent(preferredScraper->Content());
1473
1474   CFileItemList items;
1475   ADDON::AddonPtr addon;
1476   ADDON::ScraperPtr defaultScraper;
1477   if (ADDON::CAddonMgr::Get().GetDefault(type, addon))
1478     defaultScraper = boost::dynamic_pointer_cast<CScraper>(addon);
1479
1480   vector<ScraperPtr> vecScrapers;
1481
1482   // add selected scraper - first proirity
1483   if (preferredScraper)
1484     vecScrapers.push_back(preferredScraper);
1485
1486   // Add all scrapers except selected and default
1487   VECADDONS addons;
1488   CAddonMgr::Get().GetAddons(type, addons);
1489
1490   for (unsigned i = 0; i < addons.size(); ++i)
1491   {
1492     ScraperPtr scraper = boost::dynamic_pointer_cast<CScraper>(addons[i]);
1493
1494     // skip if scraper requires settings and there's nothing set yet
1495     if (!scraper || (scraper->RequiresSettings() && !scraper->HasUserSettings()))
1496       continue;
1497
1498     if((!preferredScraper || preferredScraper->ID() != scraper->ID()) && (!defaultScraper || defaultScraper->ID() != scraper->ID()) )
1499       vecScrapers.push_back(scraper);
1500   }
1501
1502   // add default scraper - not user selectable so it's last priority
1503   if(defaultScraper &&
1504      (!preferredScraper || preferredScraper->ID() != defaultScraper->ID()) &&
1505      (!defaultScraper->RequiresSettings() || defaultScraper->HasUserSettings()))
1506     vecScrapers.push_back(defaultScraper);
1507
1508   for (unsigned int i=0; i < vecScrapers.size(); ++i)
1509   {
1510     if (vecScrapers[i]->Type() != type)
1511       continue;
1512
1513     vecScrapers[i]->ClearCache();
1514     try
1515     {
1516       musicBrainzURL = vecScrapers[i]->ResolveIDToUrl(strMusicBrainzID);
1517     }
1518     catch (const ADDON::CScraperError &sce)
1519     {
1520       if (!sce.FAborted())
1521         continue;
1522     }
1523     if (!musicBrainzURL.m_url.empty())
1524     {
1525       Sleep(2000); // MusicBrainz rate-limits queries to 1 p.s - once we hit the rate-limiter
1526                    // they start serving up the 'you hit the rate-limiter' page fast - meaning
1527                    // we will never get below the rate-limit threshold again in a specific run.
1528                    // This helps us to avoidthe rate-limiter as far as possible.
1529       CLog::Log(LOGDEBUG,"-- nfo-scraper: %s",vecScrapers[i]->Name().c_str());
1530       CLog::Log(LOGDEBUG,"-- nfo url: %s", musicBrainzURL.m_url[0].m_url.c_str());
1531       musicInfoScraper.SetScraperInfo(vecScrapers[i]);
1532       bMusicBrainz = true;
1533       break;
1534     }
1535   }
1536
1537   return bMusicBrainz;
1538 }
1539
1540 map<string, string> CMusicInfoScanner::GetArtistArtwork(const CArtist& artist)
1541 {
1542   map<string, string> artwork;
1543
1544   // check thumb
1545   CStdString strFolder;
1546   CStdString thumb;
1547   if (!artist.strPath.IsEmpty())
1548   {
1549     strFolder = artist.strPath;
1550     for (int i = 0; i < 3 && thumb.IsEmpty(); ++i)
1551     {
1552       CFileItem item(strFolder, true);
1553       thumb = item.GetUserMusicThumb(true);
1554       strFolder = URIUtils::GetParentPath(strFolder);
1555     }
1556   }
1557   if (thumb.IsEmpty())
1558     thumb = CScraperUrl::GetThumbURL(artist.thumbURL.GetFirstThumb());
1559   if (!thumb.IsEmpty())
1560   {
1561     CTextureCache::Get().BackgroundCacheImage(thumb);
1562     artwork.insert(make_pair("thumb", thumb));
1563   }
1564
1565   // check fanart
1566   CStdString fanart;
1567   if (!artist.strPath.IsEmpty())
1568   {
1569     strFolder = artist.strPath;
1570     for (int i = 0; i < 3 && fanart.IsEmpty(); ++i)
1571     {
1572       CFileItem item(strFolder, true);
1573       fanart = item.GetLocalFanart();
1574       strFolder = URIUtils::GetParentPath(strFolder);
1575     }
1576   }
1577   if (fanart.IsEmpty())
1578     fanart = artist.fanart.GetImageURL();
1579   if (!fanart.IsEmpty())
1580   {
1581     CTextureCache::Get().BackgroundCacheImage(fanart);
1582     artwork.insert(make_pair("fanart", fanart));
1583   }
1584
1585   return artwork;
1586 }
1587
1588 // This function is the Run() function of the IRunnable
1589 // CFileCountReader and runs in a separate thread.
1590 void CMusicInfoScanner::Run()
1591 {
1592   int count = 0;
1593   for (set<std::string>::iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end() && !m_bStop; ++it)
1594   {
1595     count+=CountFilesRecursively(*it);
1596   }
1597   m_itemCount = count;
1598 }
1599
1600 // Recurse through all folders we scan and count files
1601 int CMusicInfoScanner::CountFilesRecursively(const CStdString& strPath)
1602 {
1603   // load subfolder
1604   CFileItemList items;
1605   CDirectory::GetDirectory(strPath, items, g_advancedSettings.m_musicExtensions, DIR_FLAG_NO_FILE_DIRS);
1606
1607   if (m_bStop)
1608     return 0;
1609
1610   // true for recursive counting
1611   int count = CountFiles(items, true);
1612   return count;
1613 }
1614
1615 int CMusicInfoScanner::CountFiles(const CFileItemList &items, bool recursive)
1616 {
1617   int count = 0;
1618   for (int i=0; i<items.Size(); ++i)
1619   {
1620     const CFileItemPtr pItem=items[i];
1621
1622     if (recursive && pItem->m_bIsFolder)
1623       count+=CountFilesRecursively(pItem->GetPath());
1624     else if (pItem->IsAudio() && !pItem->IsPlayList() && !pItem->IsNFO())
1625       count++;
1626   }
1627   return count;
1628 }