2 * Copyright (C) 2005-2013 Team XBMC
5 * This Program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2, or (at your option)
10 * This Program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with XBMC; see the file COPYING. If not, see
17 * <http://www.gnu.org/licenses/>.
21 #include "threads/SystemClock.h"
22 #include "MusicInfoScanner.h"
23 #include "music/tags/MusicInfoTagLoaderFactory.h"
24 #include "MusicAlbumInfo.h"
25 #include "MusicInfoScraper.h"
26 #include "filesystem/MusicDatabaseDirectory.h"
27 #include "filesystem/MusicDatabaseDirectory/DirectoryNode.h"
29 #include "utils/md5.h"
30 #include "GUIInfoManager.h"
31 #include "utils/Variant.h"
33 #include "music/tags/MusicInfoTag.h"
34 #include "guilib/GUIWindowManager.h"
35 #include "dialogs/GUIDialogExtendedProgressBar.h"
36 #include "dialogs/GUIDialogProgress.h"
37 #include "dialogs/GUIDialogSelect.h"
38 #include "guilib/GUIKeyboardFactory.h"
39 #include "filesystem/File.h"
40 #include "filesystem/Directory.h"
41 #include "settings/AdvancedSettings.h"
42 #include "settings/Settings.h"
44 #include "guilib/LocalizeStrings.h"
45 #include "utils/StringUtils.h"
46 #include "utils/TimeUtils.h"
47 #include "utils/log.h"
48 #include "utils/URIUtils.h"
49 #include "TextureCache.h"
50 #include "music/MusicThumbLoader.h"
51 #include "interfaces/AnnouncementManager.h"
52 #include "GUIUserMessages.h"
53 #include "addons/AddonManager.h"
54 #include "addons/Scraper.h"
59 using namespace MUSIC_INFO;
60 using namespace XFILE;
61 using namespace MUSICDATABASEDIRECTORY;
62 using namespace MUSIC_GRABBER;
63 using namespace ADDON;
65 CMusicInfoScanner::CMusicInfoScanner() : CThread("MusicInfoScanner"), m_fileCountReader(this, "MusicFileCounter")
70 m_bCanInterrupt = false;
76 CMusicInfoScanner::~CMusicInfoScanner()
80 void CMusicInfoScanner::Process()
82 ANNOUNCEMENT::CAnnouncementManager::Announce(ANNOUNCEMENT::AudioLibrary, "xbmc", "OnScanStarted");
85 unsigned int tick = XbmcThreads::SystemClockMillis();
87 m_musicDatabase.Open();
89 if (m_showDialog && !CSettings::Get().GetBool("musiclibrary.backgroundupdate"))
91 CGUIDialogExtendedProgressBar* dialog =
92 (CGUIDialogExtendedProgressBar*)g_windowManager.GetWindow(WINDOW_DIALOG_EXT_PROGRESS);
93 m_handle = dialog->GetHandle(g_localizeStrings.Get(314));
96 m_bCanInterrupt = true;
98 if (m_scanType == 0) // load info from files
100 CLog::Log(LOGDEBUG, "%s - Starting scan", __FUNCTION__);
103 m_handle->SetTitle(g_localizeStrings.Get(505));
105 // Reset progress vars
109 // Create the thread to count all files to be scanned
110 SetPriority( GetMinPriority() );
112 m_fileCountReader.Create();
114 // Database operations should not be canceled
115 // using Interupt() while scanning as it could
116 // result in unexpected behaviour.
117 m_bCanInterrupt = false;
118 m_needsCleanup = false;
121 for (std::set<std::string>::const_iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end(); it++)
123 if (!CDirectory::Exists(*it) && !m_bClean)
126 * Note that this will skip scanning (if m_bClean is disabled) if the directory really
127 * doesn't exist. Since the music scanner is fed with a list of existing paths from the DB
128 * and cleans out all songs under that path as its first step before re-adding files, if
129 * the entire source is offline we totally empty the music database in one go.
131 CLog::Log(LOGWARNING, "%s directory '%s' does not exist - skipping scan.", __FUNCTION__, it->c_str());
134 else if (!DoScan(*it))
143 g_infoManager.ResetLibraryBools();
149 m_handle->SetTitle(g_localizeStrings.Get(700));
150 m_handle->SetText("");
153 m_musicDatabase.CleanupOrphanedItems();
156 m_handle->SetTitle(g_localizeStrings.Get(331));
158 m_musicDatabase.Compress(false);
162 m_fileCountReader.StopThread();
164 m_musicDatabase.EmptyCache();
166 tick = XbmcThreads::SystemClockMillis() - tick;
167 CLog::Log(LOGNOTICE, "My Music: Scanning for music info using worker thread, operation took %s", StringUtils::SecondsToTimeString(tick / 1000).c_str());
169 if (m_scanType == 1) // load album info
171 for (std::set<std::string>::const_iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end(); ++it)
174 CDirectoryNode::GetDatabaseInfo(*it, params);
175 if (m_musicDatabase.HasAlbumBeenScraped(params.GetAlbumId())) // should this be here?
179 m_musicDatabase.GetAlbum(params.GetAlbumId(), album);
182 float percentage = (float) std::distance(it, m_pathsToScan.end()) / m_pathsToScan.size();
183 m_handle->SetText(StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator) + " - " + album.strAlbum);
184 m_handle->SetPercentage(percentage);
188 ADDON::ScraperPtr scraper;
189 if (!m_musicDatabase.GetScraperForPath(*it, scraper, ADDON::ADDON_SCRAPER_ALBUMS))
192 UpdateDatabaseAlbumInfo(album, scraper, false);
198 if (m_scanType == 2) // load artist info
200 for (std::set<std::string>::const_iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end(); ++it)
203 CDirectoryNode::GetDatabaseInfo(*it, params);
204 if (m_musicDatabase.HasArtistBeenScraped(params.GetArtistId())) // should this be here?
208 m_musicDatabase.GetArtist(params.GetArtistId(), artist);
209 m_musicDatabase.GetArtistPath(params.GetArtistId(), artist.strPath);
213 float percentage = (float) (std::distance(m_pathsToScan.begin(), it) / m_pathsToScan.size()) * 100;
214 m_handle->SetText(artist.strArtist);
215 m_handle->SetPercentage(percentage);
219 ADDON::ScraperPtr scraper;
220 if (!m_musicDatabase.GetScraperForPath(*it, scraper, ADDON::ADDON_SCRAPER_ARTISTS) || !scraper)
223 UpdateDatabaseArtistInfo(artist, scraper, false);
233 CLog::Log(LOGERROR, "MusicInfoScanner: Exception while scanning.");
235 m_musicDatabase.Close();
236 CLog::Log(LOGDEBUG, "%s - Finished scan", __FUNCTION__);
239 ANNOUNCEMENT::CAnnouncementManager::Announce(ANNOUNCEMENT::AudioLibrary, "xbmc", "OnScanFinished");
241 // we need to clear the musicdb cache and update any active lists
242 CUtil::DeleteMusicDatabaseDirectoryCache();
243 CGUIMessage msg(GUI_MSG_SCAN_FINISHED, 0, 0, 0);
244 g_windowManager.SendThreadMessage(msg);
247 m_handle->MarkFinished();
251 void CMusicInfoScanner::Start(const CStdString& strDirectory, int flags)
253 m_fileCountReader.StopThread();
255 m_pathsToScan.clear();
258 if (strDirectory.empty())
259 { // scan all paths in the database. We do this by scanning all paths in the db, and crossing them off the list as
261 m_musicDatabase.Open();
262 m_musicDatabase.GetPaths(m_pathsToScan);
263 m_musicDatabase.Close();
266 m_pathsToScan.insert(strDirectory);
267 m_bClean = g_advancedSettings.m_bMusicLibraryCleanOnUpdate;
274 void CMusicInfoScanner::FetchAlbumInfo(const CStdString& strDirectory,
277 m_fileCountReader.StopThread();
279 m_pathsToScan.clear();
282 if (strDirectory.empty())
284 m_musicDatabase.Open();
285 m_musicDatabase.GetAlbumsNav("musicdb://albums/", items);
286 m_musicDatabase.Close();
290 if (URIUtils::HasSlashAtEnd(strDirectory)) // directory
291 CDirectory::GetDirectory(strDirectory,items);
294 CFileItemPtr item(new CFileItem(strDirectory,false));
299 m_musicDatabase.Open();
300 for (int i=0;i<items.Size();++i)
302 if (CMusicDatabaseDirectory::IsAllItem(items[i]->GetPath()) || items[i]->IsParentFolder())
305 m_pathsToScan.insert(items[i]->GetPath());
308 m_musicDatabase.ClearAlbumLastScrapedTime(items[i]->GetMusicInfoTag()->GetDatabaseId());
311 m_musicDatabase.Close();
318 void CMusicInfoScanner::FetchArtistInfo(const CStdString& strDirectory,
321 m_fileCountReader.StopThread();
323 m_pathsToScan.clear();
326 if (strDirectory.empty())
328 m_musicDatabase.Open();
329 m_musicDatabase.GetArtistsNav("musicdb://artists/", items, false, -1);
330 m_musicDatabase.Close();
334 if (URIUtils::HasSlashAtEnd(strDirectory)) // directory
335 CDirectory::GetDirectory(strDirectory,items);
338 CFileItemPtr newItem(new CFileItem(strDirectory,false));
343 m_musicDatabase.Open();
344 for (int i=0;i<items.Size();++i)
346 if (CMusicDatabaseDirectory::IsAllItem(items[i]->GetPath()) || items[i]->IsParentFolder())
349 m_pathsToScan.insert(items[i]->GetPath());
352 m_musicDatabase.ClearArtistLastScrapedTime(items[i]->GetMusicInfoTag()->GetDatabaseId());
355 m_musicDatabase.Close();
362 bool CMusicInfoScanner::IsScanning()
367 void CMusicInfoScanner::Stop()
370 m_musicDatabase.Interupt();
375 static void OnDirectoryScanned(const CStdString& strDirectory)
377 CGUIMessage msg(GUI_MSG_DIRECTORY_SCANNED, 0, 0, 0);
378 msg.SetStringParam(strDirectory);
379 g_windowManager.SendThreadMessage(msg);
382 static CStdString Prettify(const CStdString& strDirectory)
384 CURL url(strDirectory);
385 CStdString strStrippedPath = url.GetWithoutUserDetails();
386 CURL::Decode(strStrippedPath);
388 return strStrippedPath;
391 bool CMusicInfoScanner::DoScan(const CStdString& strDirectory)
394 m_handle->SetText(Prettify(strDirectory));
396 // Discard all excluded files defined by m_musicExcludeRegExps
397 CStdStringArray regexps = g_advancedSettings.m_audioExcludeFromScanRegExps;
398 if (CUtil::ExcludeFileOrFolder(strDirectory, regexps))
403 CDirectory::GetDirectory(strDirectory, items, g_advancedSettings.m_musicExtensions + "|.jpg|.tbn|.lrc|.cdg");
405 // sort and get the path hash. Note that we don't filter .cue sheet items here as we want
406 // to detect changes in the .cue sheet as well. The .cue sheet items only need filtering
407 // if we have a changed hash.
408 items.Sort(SortByLabel, SortOrderAscending);
410 GetPathHash(items, hash);
412 // check whether we need to rescan or not
414 if ((m_flags & SCAN_RESCAN) || !m_musicDatabase.GetPathHash(strDirectory, dbHash) || dbHash != hash)
415 { // path has changed - rescan
417 CLog::Log(LOGDEBUG, "%s Scanning dir '%s' as not in the database", __FUNCTION__, strDirectory.c_str());
419 CLog::Log(LOGDEBUG, "%s Rescanning dir '%s' due to change", __FUNCTION__, strDirectory.c_str());
421 // filter items in the sub dir (for .cue sheet support)
422 items.FilterCueItems();
423 items.Sort(SortByLabel, SortOrderAscending);
425 // and then scan in the new information
426 if (RetrieveMusicInfo(strDirectory, items) > 0)
429 OnDirectoryScanned(strDirectory);
432 // save information about this folder
433 m_musicDatabase.SetPathHash(strDirectory, hash);
436 { // path is the same - no need to rescan
437 CLog::Log(LOGDEBUG, "%s Skipping dir '%s' due to no change", __FUNCTION__, strDirectory.c_str());
438 m_currentItem += CountFiles(items, false); // false for non-recursive
440 // updated the dialog with our progress
444 m_handle->SetPercentage(m_currentItem/(float)m_itemCount*100);
445 OnDirectoryScanned(strDirectory);
449 // now scan the subfolders
450 for (int i = 0; i < items.Size(); ++i)
452 CFileItemPtr pItem = items[i];
456 // if we have a directory item (non-playlist) we then recurse into that folder
457 if (pItem->m_bIsFolder && !pItem->IsParentFolder() && !pItem->IsPlayList())
459 CStdString strPath=pItem->GetPath();
460 if (!DoScan(strPath))
470 INFO_RET CMusicInfoScanner::ScanTags(const CFileItemList& items, CFileItemList& scannedItems)
472 CStdStringArray regexps = g_advancedSettings.m_audioExcludeFromScanRegExps;
474 for (int i = 0; i < items.Size(); ++i)
477 return INFO_CANCELLED;
479 CFileItemPtr pItem = items[i];
481 if (CUtil::ExcludeFileOrFolder(pItem->GetPath(), regexps))
484 if (pItem->m_bIsFolder || pItem->IsPlayList() || pItem->IsPicture() || pItem->IsLyrics())
489 CMusicInfoTag& tag = *pItem->GetMusicInfoTag();
492 auto_ptr<IMusicInfoTagLoader> pLoader (CMusicInfoTagLoaderFactory::CreateLoader(pItem->GetPath()));
493 if (NULL != pLoader.get())
494 pLoader->Load(pItem->GetPath(), tag);
497 if (m_handle && m_itemCount>0)
498 m_handle->SetPercentage(m_currentItem/(float)m_itemCount*100);
502 CLog::Log(LOGDEBUG, "%s - No tag found for: %s", __FUNCTION__, pItem->GetPath().c_str());
505 scannedItems.Add(pItem);
510 static bool SortSongsByTrack(const CSong& song, const CSong& song2)
512 return song.iTrack < song2.iTrack;
515 void CMusicInfoScanner::FileItemsToAlbums(CFileItemList& items, VECALBUMS& albums, MAPSONGS* songsMap /* = NULL */)
518 * Step 1: Convert the FileItems into Songs.
519 * If they're MB tagged, create albums directly from the FileItems.
520 * If they're non-MB tagged, index them by album name ready for step 2.
522 map<string, VECSONGS> songsByAlbumNames;
523 for (int i = 0; i < items.Size(); ++i)
525 CMusicInfoTag& tag = *items[i]->GetMusicInfoTag();
526 CSong song(*items[i]);
528 // keep the db-only fields intact on rescan...
529 if (songsMap != NULL)
531 MAPSONGS::iterator it = songsMap->find(items[i]->GetPath());
532 if (it != songsMap->end())
534 song.iTimesPlayed = it->second.iTimesPlayed;
535 song.lastPlayed = it->second.lastPlayed;
536 song.iKaraokeNumber = it->second.iKaraokeNumber;
537 if (song.rating == '0') song.rating = it->second.rating;
538 if (song.strThumb.empty()) song.strThumb = it->second.strThumb;
542 if (!tag.GetMusicBrainzAlbumID().empty())
544 VECALBUMS::iterator it;
545 for (it = albums.begin(); it != albums.end(); ++it)
546 if (it->strMusicBrainzAlbumID.Equals(tag.GetMusicBrainzAlbumID()))
549 if (it == albums.end())
551 CAlbum album(*items[i]);
552 album.songs.push_back(song);
553 albums.push_back(album);
556 it->songs.push_back(song);
559 songsByAlbumNames[tag.GetAlbum()].push_back(song);
563 Step 2: Split into unique albums based on album name and album artist
564 In the case where the album artist is unknown, we use the primary artist
565 (i.e. first artist from each song).
567 for (map<string, VECSONGS>::iterator songsByAlbumName = songsByAlbumNames.begin(); songsByAlbumName != songsByAlbumNames.end(); ++songsByAlbumName)
569 VECSONGS &songs = songsByAlbumName->second;
570 // sort the songs by tracknumber to identify duplicate track numbers
571 sort(songs.begin(), songs.end(), SortSongsByTrack);
573 // map the songs to their primary artists
574 bool tracksOverlap = false;
575 bool hasAlbumArtist = false;
576 bool isCompilation = true;
578 map<string, vector<CSong *> > artists;
579 for (VECSONGS::iterator song = songs.begin(); song != songs.end(); ++song)
581 // test for song overlap
582 if (song != songs.begin() && song->iTrack == (song - 1)->iTrack)
583 tracksOverlap = true;
585 if (!song->bCompilation)
586 isCompilation = false;
588 // get primary artist
590 if (!song->albumArtist.empty())
592 primary = song->albumArtist[0];
593 hasAlbumArtist = true;
595 else if (!song->artist.empty())
596 primary = song->artist[0];
598 // add to the artist map
599 artists[primary].push_back(&(*song));
603 We have a compilation if
604 1. album name is non-empty AND
605 2a. no tracks overlap OR
606 2b. all tracks are marked as part of compilation AND
607 3a. a unique primary artist is specified as "various" or "various artists" OR
608 3b. we have at least two primary artists and no album artist specified.
610 bool compilation = !songsByAlbumName->first.empty() && (isCompilation || !tracksOverlap); // 1+2b+2a
611 if (artists.size() == 1)
613 string artist = artists.begin()->first; StringUtils::ToLower(artist);
614 if (!StringUtils::EqualsNoCase(artist, "various") &&
615 !StringUtils::EqualsNoCase(artist, "various artists")) // 3a
618 else if (hasAlbumArtist) // 3b
623 CLog::Log(LOGDEBUG, "Album '%s' is a compilation as there's no overlapping tracks and %s", songsByAlbumName->first.c_str(), hasAlbumArtist ? "the album artist is 'Various'" : "there is more than one unique artist");
625 std::string various = g_localizeStrings.Get(340); // Various Artists
626 vector<string> va; va.push_back(various);
627 for (VECSONGS::iterator song = songs.begin(); song != songs.end(); ++song)
629 song->albumArtist = va;
630 artists[various].push_back(&(*song));
635 Step 3: Find the common albumartist for each song and assign
636 albumartist to those tracks that don't have it set.
638 for (map<string, vector<CSong *> >::iterator j = artists.begin(); j != artists.end(); ++j)
640 // find the common artist for these songs
641 vector<CSong *> &artistSongs = j->second;
642 vector<string> common = artistSongs.front()->albumArtist.empty() ? artistSongs.front()->artist : artistSongs.front()->albumArtist;
643 for (vector<CSong *>::iterator k = artistSongs.begin() + 1; k != artistSongs.end(); ++k)
645 unsigned int match = 0;
646 vector<string> &compare = (*k)->albumArtist.empty() ? (*k)->artist : (*k)->albumArtist;
647 for (; match < common.size() && match < compare.size(); match++)
649 if (compare[match] != common[match])
652 common.erase(common.begin() + match, common.end());
656 Step 4: Assign the album artist for each song that doesn't have it set
657 and add to the album vector
660 album.strAlbum = songsByAlbumName->first;
661 album.artist = common;
662 for (vector<string>::iterator it = common.begin(); it != common.end(); ++it)
664 CStdString strJoinPhrase = (it == --common.end() ? "" : g_advancedSettings.m_musicItemSeparator);
665 CArtistCredit artistCredit(*it, strJoinPhrase);
666 album.artistCredits.push_back(artistCredit);
668 album.bCompilation = compilation;
669 for (vector<CSong *>::iterator k = artistSongs.begin(); k != artistSongs.end(); ++k)
671 if ((*k)->albumArtist.empty())
672 (*k)->albumArtist = common;
673 // TODO: in future we may wish to union up the genres, for now we assume they're the same
674 album.genre = (*k)->genre;
675 // in addition, we may want to use year as discriminating for albums
676 album.iYear = (*k)->iYear;
677 album.songs.push_back(**k);
679 albums.push_back(album);
684 int CMusicInfoScanner::RetrieveMusicInfo(const CStdString& strDirectory, CFileItemList& items)
688 // get all information for all files in current directory from database, and remove them
689 if (m_musicDatabase.RemoveSongsFromPath(strDirectory, songsMap))
690 m_needsCleanup = true;
692 CFileItemList scannedItems;
693 if (ScanTags(items, scannedItems) == INFO_CANCELLED || scannedItems.Size() == 0)
697 FileItemsToAlbums(scannedItems, albums, &songsMap);
698 FindArtForAlbums(albums, items.GetPath());
701 ADDON::AddonPtr addon;
702 ADDON::ScraperPtr albumScraper;
703 ADDON::ScraperPtr artistScraper;
704 if(ADDON::CAddonMgr::Get().GetDefault(ADDON::ADDON_SCRAPER_ALBUMS, addon))
705 albumScraper = boost::dynamic_pointer_cast<ADDON::CScraper>(addon);
707 if(ADDON::CAddonMgr::Get().GetDefault(ADDON::ADDON_SCRAPER_ARTISTS, addon))
708 artistScraper = boost::dynamic_pointer_cast<ADDON::CScraper>(addon);
711 for (VECALBUMS::iterator album = albums.begin(); album != albums.end(); ++album)
716 album->strPath = strDirectory;
717 m_musicDatabase.AddAlbum(*album);
719 // Yuk - this is a kludgy way to do what we want to do, but it will work to sort
720 // out artist fanart until we can restructure the artist fanart to work more
721 // like the album fanart. This has to be done after we've added the album so
722 // we have the artist IDs to update, but before we call UpdateDatabaseArtistInfo.
723 if (albums.size() == 1 &&
724 album->artistCredits.size() > 0 &&
725 !StringUtils::EqualsNoCase(album->artistCredits[0].GetArtist(), "various artists") &&
726 !StringUtils::EqualsNoCase(album->artistCredits[0].GetArtist(), "various"))
729 if (m_musicDatabase.GetArtist(album->artistCredits[0].GetArtistId(), artist))
731 artist.strPath = URIUtils::GetParentPath(strDirectory);
732 m_musicDatabase.SetArtForItem(artist.idArtist, "artist", GetArtistArtwork(artist));
736 if ((m_flags & SCAN_ONLINE))
738 if (!albumScraper || !artistScraper)
741 INFO_RET albumScrapeStatus = INFO_NOT_FOUND;
742 if (!m_musicDatabase.HasAlbumBeenScraped(album->idAlbum))
743 albumScrapeStatus = UpdateDatabaseAlbumInfo(*album, albumScraper, false);
745 if (albumScrapeStatus == INFO_ADDED)
747 for (VECARTISTCREDITS::const_iterator artistCredit = album->artistCredits.begin();
748 artistCredit != album->artistCredits.end();
754 if (!m_musicDatabase.HasArtistBeenScraped(artistCredit->GetArtistId()))
757 m_musicDatabase.GetArtist(artistCredit->GetArtistId(), artist);
758 UpdateDatabaseArtistInfo(artist, artistScraper, false);
762 for (VECSONGS::iterator song = album->songs.begin();
763 song != album->songs.end();
769 for (VECARTISTCREDITS::const_iterator artistCredit = song->artistCredits.begin();
770 artistCredit != song->artistCredits.end();
776 CMusicArtistInfo musicArtistInfo;
777 if (!m_musicDatabase.HasArtistBeenScraped(artistCredit->GetArtistId()))
780 m_musicDatabase.GetArtist(artistCredit->GetArtistId(), artist);
781 UpdateDatabaseArtistInfo(artist, artistScraper, false);
787 numAdded += album->songs.size();
791 m_handle->SetTitle(g_localizeStrings.Get(505));
796 void CMusicInfoScanner::FindArtForAlbums(VECALBUMS &albums, const CStdString &path)
799 If there's a single album in the folder, then art can be taken from
802 std::string albumArt;
803 if (albums.size() == 1)
805 CFileItem album(path, true);
806 albumArt = album.GetUserMusicThumb(true);
807 if (!albumArt.empty())
808 albums[0].art["thumb"] = albumArt;
810 for (VECALBUMS::iterator i = albums.begin(); i != albums.end(); ++i)
814 if (albums.size() != 1)
818 Find art that is common across these items
819 If we find a single art image we treat it as the album art
820 and discard song art else we use first as album art and
821 keep everything as song art.
823 bool singleArt = true;
825 for (VECSONGS::iterator k = album.songs.begin(); k != album.songs.end(); ++k)
830 if (art && !art->ArtMatches(song))
841 assign the first art found to the album - better than no art at all
844 if (art && albumArt.empty())
846 if (!art->strThumb.empty())
847 albumArt = art->strThumb;
849 albumArt = CTextureUtils::GetWrappedImageURL(art->strFileName, "music");
852 if (!albumArt.empty())
853 album.art["thumb"] = albumArt;
856 { //if singleArt then we can clear the artwork for all songs
857 for (VECSONGS::iterator k = album.songs.begin(); k != album.songs.end(); ++k)
861 { // more than one piece of art was found for these songs, so cache per song
862 for (VECSONGS::iterator k = album.songs.begin(); k != album.songs.end(); ++k)
864 if (k->strThumb.empty() && !k->embeddedArt.empty())
865 k->strThumb = CTextureUtils::GetWrappedImageURL(k->strFileName, "music");
869 if (albums.size() == 1 && !albumArt.empty())
871 // assign to folder thumb as well
872 CFileItem albumItem(path, true);
873 CMusicThumbLoader loader;
874 loader.SetCachedImage(albumItem, "thumb", albumArt);
878 int CMusicInfoScanner::GetPathHash(const CFileItemList &items, CStdString &hash)
880 // Create a hash based on the filenames, filesize and filedate. Also count the number of files
881 if (0 == items.Size()) return 0;
882 XBMC::XBMC_MD5 md5state;
884 for (int i = 0; i < items.Size(); ++i)
886 const CFileItemPtr pItem = items[i];
887 md5state.append(pItem->GetPath());
888 md5state.append((unsigned char *)&pItem->m_dwSize, sizeof(pItem->m_dwSize));
889 FILETIME time = pItem->m_dateTime;
890 md5state.append((unsigned char *)&time, sizeof(FILETIME));
891 if (pItem->IsAudio() && !pItem->IsPlayList() && !pItem->IsNFO())
894 md5state.getDigest(hash);
898 INFO_RET CMusicInfoScanner::UpdateDatabaseAlbumInfo(CAlbum& album, const ADDON::ScraperPtr& scraper, bool bAllowSelection, CGUIDialogProgress* pDialog /* = NULL */)
903 CMusicAlbumInfo albumInfo;
906 CLog::Log(LOGDEBUG, "%s downloading info for: %s", __FUNCTION__, album.strAlbum.c_str());
907 INFO_RET albumDownloadStatus = DownloadAlbumInfo(album, scraper, albumInfo, pDialog);
908 if (albumDownloadStatus == INFO_NOT_FOUND)
910 if (pDialog && bAllowSelection)
912 if (!CGUIKeyboardFactory::ShowAndGetInput(album.strAlbum, g_localizeStrings.Get(16011), false))
913 return INFO_CANCELLED;
915 CStdString strTempArtist(StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
916 if (!CGUIKeyboardFactory::ShowAndGetInput(strTempArtist, g_localizeStrings.Get(16025), false))
917 return INFO_CANCELLED;
919 album.artist = StringUtils::Split(strTempArtist, g_advancedSettings.m_musicItemSeparator);
923 else if (albumDownloadStatus == INFO_ADDED)
925 album.MergeScrapedAlbum(albumInfo.GetAlbum(), CSettings::Get().GetBool("musiclibrary.overridetags"));
926 m_musicDatabase.Open();
927 m_musicDatabase.UpdateAlbum(album);
928 GetAlbumArtwork(album.idAlbum, album);
929 m_musicDatabase.Close();
930 albumInfo.SetLoaded(true);
932 return albumDownloadStatus;
935 INFO_RET CMusicInfoScanner::UpdateDatabaseArtistInfo(CArtist& artist, const ADDON::ScraperPtr& scraper, bool bAllowSelection, CGUIDialogProgress* pDialog /* = NULL */)
940 CMusicArtistInfo artistInfo;
943 CLog::Log(LOGDEBUG, "%s downloading info for: %s", __FUNCTION__, artist.strArtist.c_str());
944 INFO_RET artistDownloadStatus = DownloadArtistInfo(artist, scraper, artistInfo, pDialog);
945 if (artistDownloadStatus == INFO_NOT_FOUND)
947 if (pDialog && bAllowSelection)
949 if (!CGUIKeyboardFactory::ShowAndGetInput(artist.strArtist, g_localizeStrings.Get(16025), false))
950 return INFO_CANCELLED;
954 else if (artistDownloadStatus == INFO_ADDED)
956 artist.MergeScrapedArtist(artistInfo.GetArtist(), CSettings::Get().GetBool("musiclibrary.overridetags"));
957 m_musicDatabase.Open();
958 m_musicDatabase.UpdateArtist(artist);
959 m_musicDatabase.GetArtistPath(artist.idArtist, artist.strPath);
960 m_musicDatabase.SetArtForItem(artist.idArtist, "artist", GetArtistArtwork(artist));
961 m_musicDatabase.Close();
962 artistInfo.SetLoaded();
964 return artistDownloadStatus;
967 #define THRESHOLD .95f
969 INFO_RET CMusicInfoScanner::DownloadAlbumInfo(const CAlbum& album, const ADDON::ScraperPtr& info, CMusicAlbumInfo& albumInfo, CGUIDialogProgress* pDialog)
973 m_handle->SetTitle(StringUtils::Format(g_localizeStrings.Get(20321), info->Name().c_str()));
974 m_handle->SetText(StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator) + " - " + album.strAlbum);
977 // clear our scraper cache
980 CMusicInfoScraper scraper(info);
981 bool bMusicBrainz = false;
982 if (!album.strMusicBrainzAlbumID.empty())
984 CScraperUrl musicBrainzURL;
985 if (ResolveMusicBrainz(album.strMusicBrainzAlbumID, info, musicBrainzURL))
987 CMusicAlbumInfo albumNfo("nfo", musicBrainzURL);
988 scraper.GetAlbums().clear();
989 scraper.GetAlbums().push_back(albumNfo);
995 CStdString strNfo = URIUtils::AddFileToFolder(album.strPath, "album.nfo");
996 CNfoFile::NFOResult result = CNfoFile::NO_NFO;
998 if (XFILE::CFile::Exists(strNfo))
1000 CLog::Log(LOGDEBUG,"Found matching nfo file: %s", strNfo.c_str());
1001 result = nfoReader.Create(strNfo, info, -1, album.strPath);
1002 if (result == CNfoFile::FULL_NFO)
1004 CLog::Log(LOGDEBUG, "%s Got details from nfo", __FUNCTION__);
1005 nfoReader.GetDetails(albumInfo.GetAlbum());
1008 else if (result == CNfoFile::URL_NFO || result == CNfoFile::COMBINED_NFO)
1010 CScraperUrl scrUrl(nfoReader.ScraperUrl());
1011 CMusicAlbumInfo albumNfo("nfo",scrUrl);
1012 ADDON::ScraperPtr nfoReaderScraper = nfoReader.GetScraperInfo();
1013 CLog::Log(LOGDEBUG,"-- nfo-scraper: %s", nfoReaderScraper->Name().c_str());
1014 CLog::Log(LOGDEBUG,"-- nfo url: %s", scrUrl.m_url[0].m_url.c_str());
1015 scraper.SetScraperInfo(nfoReaderScraper);
1016 scraper.GetAlbums().clear();
1017 scraper.GetAlbums().push_back(albumNfo);
1020 CLog::Log(LOGERROR,"Unable to find an url in nfo file: %s", strNfo.c_str());
1023 if (!scraper.CheckValidOrFallback(CSettings::Get().GetString("musiclibrary.albumsscraper")))
1024 { // the current scraper is invalid, as is the default - bail
1025 CLog::Log(LOGERROR, "%s - current and default scrapers are invalid. Pick another one", __FUNCTION__);
1029 if (!scraper.GetAlbumCount())
1031 scraper.FindAlbumInfo(album.strAlbum, StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
1033 while (!scraper.Completed())
1038 return INFO_CANCELLED;
1044 CGUIDialogSelect *pDlg = NULL;
1045 int iSelectedAlbum=0;
1046 if (result == CNfoFile::NO_NFO && !bMusicBrainz)
1048 iSelectedAlbum = -1; // set negative so that we can detect a failure
1049 if (scraper.Succeeded() && scraper.GetAlbumCount() >= 1)
1051 double bestRelevance = 0;
1052 double minRelevance = THRESHOLD;
1053 if (scraper.GetAlbumCount() > 1) // score the matches
1055 //show dialog with all albums found
1058 pDlg = (CGUIDialogSelect*)g_windowManager.GetWindow(WINDOW_DIALOG_SELECT);
1059 pDlg->SetHeading(g_localizeStrings.Get(181).c_str());
1061 pDlg->EnableButton(true, 413); // manual
1064 for (int i = 0; i < scraper.GetAlbumCount(); ++i)
1066 CMusicAlbumInfo& info = scraper.GetAlbum(i);
1067 double relevance = info.GetRelevance();
1069 relevance = CUtil::AlbumRelevance(info.GetAlbum().strAlbum, album.strAlbum, StringUtils::Join(info.GetAlbum().artist, g_advancedSettings.m_musicItemSeparator), StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
1071 // if we're doing auto-selection (ie querying all albums at once, then allow 95->100% for perfect matches)
1072 // otherwise, perfect matches only
1073 if (relevance >= max(minRelevance, bestRelevance))
1074 { // we auto-select the best of these
1075 bestRelevance = relevance;
1080 // set the label to [relevance] album - artist
1081 CStdString strTemp = StringUtils::Format("[%0.2f] %s", relevance, info.GetTitle2().c_str());
1082 CFileItem item(strTemp);
1083 item.m_idepth = i; // use this to hold the index of the album in the scraper
1086 if (relevance > .99f) // we're so close, no reason to search further
1090 if (pDialog && bestRelevance < THRESHOLD)
1095 // and wait till user selects one
1096 if (pDlg->GetSelectedLabel() < 0)
1098 if (!pDlg->IsButtonPressed())
1099 return INFO_CANCELLED;
1101 // manual button pressed
1102 CStdString strNewAlbum = album.strAlbum;
1103 if (!CGUIKeyboardFactory::ShowAndGetInput(strNewAlbum, g_localizeStrings.Get(16011), false)) return INFO_CANCELLED;
1104 if (strNewAlbum == "") return INFO_CANCELLED;
1106 CStdString strNewArtist = StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator);
1107 if (!CGUIKeyboardFactory::ShowAndGetInput(strNewArtist, g_localizeStrings.Get(16025), false)) return INFO_CANCELLED;
1109 pDialog->SetLine(0, strNewAlbum);
1110 pDialog->SetLine(1, strNewArtist);
1111 pDialog->Progress();
1113 CAlbum newAlbum = album;
1114 newAlbum.strAlbum = strNewAlbum;
1115 newAlbum.artist = StringUtils::Split(strNewArtist, g_advancedSettings.m_musicItemSeparator);
1117 return DownloadAlbumInfo(newAlbum, info, albumInfo, pDialog);
1119 iSelectedAlbum = pDlg->GetSelectedItem()->m_idepth;
1124 CMusicAlbumInfo& info = scraper.GetAlbum(0);
1125 double relevance = info.GetRelevance();
1127 relevance = CUtil::AlbumRelevance(info.GetAlbum().strAlbum,
1129 StringUtils::Join(info.GetAlbum().artist, g_advancedSettings.m_musicItemSeparator),
1130 StringUtils::Join(album.artist, g_advancedSettings.m_musicItemSeparator));
1131 if (relevance < THRESHOLD)
1132 return INFO_NOT_FOUND;
1138 if (iSelectedAlbum < 0)
1139 return INFO_NOT_FOUND;
1143 scraper.LoadAlbumInfo(iSelectedAlbum);
1144 while (!scraper.Completed())
1149 return INFO_CANCELLED;
1154 if (!scraper.Succeeded())
1157 albumInfo = scraper.GetAlbum(iSelectedAlbum);
1159 if (result == CNfoFile::COMBINED_NFO)
1160 nfoReader.GetDetails(albumInfo.GetAlbum(), NULL, true);
1165 void CMusicInfoScanner::GetAlbumArtwork(long id, const CAlbum &album)
1167 if (album.thumbURL.m_url.size())
1169 if (m_musicDatabase.GetArtForItem(id, "album", "thumb").empty())
1171 string thumb = CScraperUrl::GetThumbURL(album.thumbURL.GetFirstThumb());
1174 CTextureCache::Get().BackgroundCacheImage(thumb);
1175 m_musicDatabase.SetArtForItem(id, "album", "thumb", thumb);
1181 INFO_RET CMusicInfoScanner::DownloadArtistInfo(const CArtist& artist, const ADDON::ScraperPtr& info, MUSIC_GRABBER::CMusicArtistInfo& artistInfo, CGUIDialogProgress* pDialog)
1185 m_handle->SetTitle(StringUtils::Format(g_localizeStrings.Get(20320), info->Name().c_str()));
1186 m_handle->SetText(artist.strArtist);
1189 // clear our scraper cache
1192 CMusicInfoScraper scraper(info);
1193 bool bMusicBrainz = false;
1194 if (!artist.strMusicBrainzArtistID.empty())
1196 CScraperUrl musicBrainzURL;
1197 if (ResolveMusicBrainz(artist.strMusicBrainzArtistID, info, musicBrainzURL))
1199 CMusicArtistInfo artistNfo("nfo", musicBrainzURL);
1200 scraper.GetArtists().clear();
1201 scraper.GetArtists().push_back(artistNfo);
1202 bMusicBrainz = true;
1207 CStdString strNfo = URIUtils::AddFileToFolder(artist.strPath, "artist.nfo");
1208 CNfoFile::NFOResult result=CNfoFile::NO_NFO;
1210 if (XFILE::CFile::Exists(strNfo))
1212 CLog::Log(LOGDEBUG,"Found matching nfo file: %s", strNfo.c_str());
1213 result = nfoReader.Create(strNfo, info);
1214 if (result == CNfoFile::FULL_NFO)
1216 CLog::Log(LOGDEBUG, "%s Got details from nfo", __FUNCTION__);
1217 nfoReader.GetDetails(artistInfo.GetArtist());
1220 else if (result == CNfoFile::URL_NFO || result == CNfoFile::COMBINED_NFO)
1222 CScraperUrl scrUrl(nfoReader.ScraperUrl());
1223 CMusicArtistInfo artistNfo("nfo",scrUrl);
1224 ADDON::ScraperPtr nfoReaderScraper = nfoReader.GetScraperInfo();
1225 CLog::Log(LOGDEBUG,"-- nfo-scraper: %s",nfoReaderScraper->Name().c_str());
1226 CLog::Log(LOGDEBUG,"-- nfo url: %s", scrUrl.m_url[0].m_url.c_str());
1227 scraper.SetScraperInfo(nfoReaderScraper);
1228 scraper.GetArtists().push_back(artistNfo);
1231 CLog::Log(LOGERROR,"Unable to find an url in nfo file: %s", strNfo.c_str());
1234 if (!scraper.GetArtistCount())
1236 scraper.FindArtistInfo(artist.strArtist);
1238 while (!scraper.Completed())
1243 return INFO_CANCELLED;
1249 int iSelectedArtist = 0;
1250 if (result == CNfoFile::NO_NFO && !bMusicBrainz)
1252 if (scraper.GetArtistCount() >= 1)
1254 // now load the first match
1255 if (pDialog && scraper.GetArtistCount() > 1)
1257 // if we found more then 1 album, let user choose one
1258 CGUIDialogSelect *pDlg = (CGUIDialogSelect*)g_windowManager.GetWindow(WINDOW_DIALOG_SELECT);
1261 pDlg->SetHeading(g_localizeStrings.Get(21890));
1263 pDlg->EnableButton(true, 413); // manual
1265 for (int i = 0; i < scraper.GetArtistCount(); ++i)
1267 // set the label to artist
1268 CFileItem item(scraper.GetArtist(i).GetArtist());
1269 CStdString strTemp=scraper.GetArtist(i).GetArtist().strArtist;
1270 if (!scraper.GetArtist(i).GetArtist().strBorn.empty())
1271 strTemp += " ("+scraper.GetArtist(i).GetArtist().strBorn+")";
1272 if (!scraper.GetArtist(i).GetArtist().genre.empty())
1274 CStdString genres = StringUtils::Join(scraper.GetArtist(i).GetArtist().genre, g_advancedSettings.m_musicItemSeparator);
1275 if (!genres.empty())
1276 strTemp = StringUtils::Format("[%s] %s", genres.c_str(), strTemp.c_str());
1278 item.SetLabel(strTemp);
1279 item.m_idepth = i; // use this to hold the index of the album in the scraper
1284 // and wait till user selects one
1285 if (pDlg->GetSelectedLabel() < 0)
1287 if (!pDlg->IsButtonPressed())
1288 return INFO_CANCELLED;
1290 // manual button pressed
1291 CStdString strNewArtist = artist.strArtist;
1292 if (!CGUIKeyboardFactory::ShowAndGetInput(strNewArtist, g_localizeStrings.Get(16025), false)) return INFO_CANCELLED;
1296 pDialog->SetLine(0, strNewArtist);
1297 pDialog->Progress();
1301 newArtist.strArtist = strNewArtist;
1302 return DownloadArtistInfo(newArtist, info, artistInfo, pDialog);
1304 iSelectedArtist = pDlg->GetSelectedItem()->m_idepth;
1309 return INFO_NOT_FOUND;
1312 scraper.LoadArtistInfo(iSelectedArtist, artist.strArtist);
1313 while (!scraper.Completed())
1318 return INFO_CANCELLED;
1323 if (!scraper.Succeeded())
1326 artistInfo = scraper.GetArtist(iSelectedArtist);
1328 if (result == CNfoFile::COMBINED_NFO)
1329 nfoReader.GetDetails(artistInfo.GetArtist(), NULL, true);
1334 bool CMusicInfoScanner::ResolveMusicBrainz(const CStdString &strMusicBrainzID, const ScraperPtr &preferredScraper, CScraperUrl &musicBrainzURL)
1336 // We have a MusicBrainz ID
1337 // Get a scraper that can resolve it to a MusicBrainz URL & force our
1338 // search directly to the specific album.
1339 bool bMusicBrainz = false;
1342 musicBrainzURL = preferredScraper->ResolveIDToUrl(strMusicBrainzID);
1344 catch (const ADDON::CScraperError &sce)
1350 if (!musicBrainzURL.m_url.empty())
1352 Sleep(2000); // MusicBrainz rate-limits queries to 1 p.s - once we hit the rate-limiter
1353 // they start serving up the 'you hit the rate-limiter' page fast - meaning
1354 // we will never get below the rate-limit threshold again in a specific run.
1355 // This helps us to avoidthe rate-limiter as far as possible.
1356 CLog::Log(LOGDEBUG,"-- nfo-scraper: %s",preferredScraper->Name().c_str());
1357 CLog::Log(LOGDEBUG,"-- nfo url: %s", musicBrainzURL.m_url[0].m_url.c_str());
1358 bMusicBrainz = true;
1361 return bMusicBrainz;
1364 map<string, string> CMusicInfoScanner::GetArtistArtwork(const CArtist& artist)
1366 map<string, string> artwork;
1369 CStdString strFolder;
1371 if (!artist.strPath.empty())
1373 strFolder = artist.strPath;
1374 for (int i = 0; i < 3 && thumb.empty(); ++i)
1376 CFileItem item(strFolder, true);
1377 thumb = item.GetUserMusicThumb(true);
1378 strFolder = URIUtils::GetParentPath(strFolder);
1382 thumb = CScraperUrl::GetThumbURL(artist.thumbURL.GetFirstThumb());
1385 CTextureCache::Get().BackgroundCacheImage(thumb);
1386 artwork.insert(make_pair("thumb", thumb));
1391 if (!artist.strPath.empty())
1393 strFolder = artist.strPath;
1394 for (int i = 0; i < 3 && fanart.empty(); ++i)
1396 CFileItem item(strFolder, true);
1397 fanart = item.GetLocalFanart();
1398 strFolder = URIUtils::GetParentPath(strFolder);
1402 fanart = artist.fanart.GetImageURL();
1403 if (!fanart.empty())
1405 CTextureCache::Get().BackgroundCacheImage(fanart);
1406 artwork.insert(make_pair("fanart", fanart));
1412 // This function is the Run() function of the IRunnable
1413 // CFileCountReader and runs in a separate thread.
1414 void CMusicInfoScanner::Run()
1417 for (set<std::string>::iterator it = m_pathsToScan.begin(); it != m_pathsToScan.end() && !m_bStop; ++it)
1419 count+=CountFilesRecursively(*it);
1421 m_itemCount = count;
1424 // Recurse through all folders we scan and count files
1425 int CMusicInfoScanner::CountFilesRecursively(const CStdString& strPath)
1428 CFileItemList items;
1429 CDirectory::GetDirectory(strPath, items, g_advancedSettings.m_musicExtensions, DIR_FLAG_NO_FILE_DIRS);
1434 // true for recursive counting
1435 int count = CountFiles(items, true);
1439 int CMusicInfoScanner::CountFiles(const CFileItemList &items, bool recursive)
1442 for (int i=0; i<items.Size(); ++i)
1444 const CFileItemPtr pItem=items[i];
1446 if (recursive && pItem->m_bIsFolder)
1447 count+=CountFilesRecursively(pItem->GetPath());
1448 else if (pItem->IsAudio() && !pItem->IsPlayList() && !pItem->IsNFO())