2 * Copyright (C) 2006, 2007, 2008, 2009, 2011 Apple Inc. All rights reserved.
3 * Copyright (C) 2007 Justin Haygood (jhaygood@reaktix.com)
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
15 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE COMPUTER, INC. OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
21 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
22 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
24 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 #include "IconDatabase.h"
30 #if ENABLE(ICONDATABASE)
32 #include "AutodrainedPool.h"
33 #include "DocumentLoader.h"
34 #include "FileSystem.h"
35 #include "IconDatabaseClient.h"
36 #include "IconRecord.h"
39 #include "SQLiteStatement.h"
40 #include "SQLiteTransaction.h"
41 #include "SuddenTermination.h"
42 #include <wtf/CurrentTime.h>
43 #include <wtf/MainThread.h>
44 #include <wtf/StdLibExtras.h>
45 #include <wtf/text/CString.h>
47 // For methods that are meant to support API from the main thread - should not be called internally
48 #define ASSERT_NOT_SYNC_THREAD() ASSERT(!m_syncThreadRunning || !IS_ICON_SYNC_THREAD())
50 // For methods that are meant to support the sync thread ONLY
51 #define IS_ICON_SYNC_THREAD() (m_syncThread == currentThread())
52 #define ASSERT_ICON_SYNC_THREAD() ASSERT(IS_ICON_SYNC_THREAD())
54 #if PLATFORM(QT) || PLATFORM(GTK)
55 #define CAN_THEME_URL_ICON
60 static int databaseCleanupCounter = 0;
62 // This version number is in the DB and marks the current generation of the schema
63 // Currently, a mismatched schema causes the DB to be wiped and reset. This isn't
64 // so bad during development but in the future, we would need to write a conversion
65 // function to advance older released schemas to "current"
66 static const int currentDatabaseVersion = 6;
68 // Icons expire once every 4 days
69 static const int iconExpirationTime = 60*60*24*4;
71 static const int updateTimerDelay = 5;
73 static bool checkIntegrityOnOpen = false;
76 static String urlForLogging(const String& url)
78 static unsigned urlTruncationLength = 120;
80 if (url.length() < urlTruncationLength)
82 return url.substring(0, urlTruncationLength) + "...";
86 class DefaultIconDatabaseClient : public IconDatabaseClient {
88 virtual bool performImport() { return true; }
89 virtual void didImportIconURLForPageURL(const String&) { }
90 virtual void didImportIconDataForPageURL(const String&) { }
91 virtual void didChangeIconForPageURL(const String&) { }
92 virtual void didRemoveAllIcons() { }
93 virtual void didFinishURLImport() { }
96 static IconDatabaseClient* defaultClient()
98 static IconDatabaseClient* defaultClient = new DefaultIconDatabaseClient();
102 // ************************
103 // *** Main Thread Only ***
104 // ************************
106 void IconDatabase::setClient(IconDatabaseClient* client)
108 // We don't allow a null client, because we never null check it anywhere in this code
109 // Also don't allow a client change after the thread has already began
110 // (setting the client should occur before the database is opened)
112 ASSERT(!m_syncThreadRunning);
113 if (!client || m_syncThreadRunning)
119 bool IconDatabase::open(const String& directory, const String& filename)
121 ASSERT_NOT_SYNC_THREAD();
127 LOG_ERROR("Attempt to reopen the IconDatabase which is already open. Must close it first.");
131 m_databaseDirectory = directory.crossThreadString();
133 // Formulate the full path for the database file
134 m_completeDatabasePath = pathByAppendingComponent(m_databaseDirectory, filename);
136 // Lock here as well as first thing in the thread so the thread doesn't actually commence until the createThread() call
137 // completes and m_syncThreadRunning is properly set
139 m_syncThread = createThread(IconDatabase::iconDatabaseSyncThreadStart, this, "WebCore: IconDatabase");
140 m_syncThreadRunning = m_syncThread;
147 void IconDatabase::close()
149 ASSERT_NOT_SYNC_THREAD();
151 if (m_syncThreadRunning) {
152 // Set the flag to tell the sync thread to wrap it up
153 m_threadTerminationRequested = true;
155 // Wake up the sync thread if it's waiting
158 // Wait for the sync thread to terminate
159 waitForThreadCompletion(m_syncThread, 0);
162 m_syncThreadRunning = false;
163 m_threadTerminationRequested = false;
164 m_removeIconsRequested = false;
170 void IconDatabase::removeAllIcons()
172 ASSERT_NOT_SYNC_THREAD();
177 LOG(IconDatabase, "Requesting background thread to remove all icons");
179 // Clear the in-memory record of every IconRecord, anything waiting to be read from disk, and anything waiting to be written to disk
181 MutexLocker locker(m_urlAndIconLock);
183 // Clear the IconRecords for every page URL - RefCounting will cause the IconRecords themselves to be deleted
184 // We don't delete the actual PageRecords because we have the "retain icon for url" count to keep track of
185 HashMap<String, PageURLRecord*>::iterator iter = m_pageURLToRecordMap.begin();
186 HashMap<String, PageURLRecord*>::iterator end = m_pageURLToRecordMap.end();
187 for (; iter != end; ++iter)
188 (*iter).second->setIconRecord(0);
190 // Clear the iconURL -> IconRecord map
191 m_iconURLToRecordMap.clear();
193 // Clear all in-memory records of things that need to be synced out to disk
195 MutexLocker locker(m_pendingSyncLock);
196 m_pageURLsPendingSync.clear();
197 m_iconsPendingSync.clear();
200 // Clear all in-memory records of things that need to be read in from disk
202 MutexLocker locker(m_pendingReadingLock);
203 m_pageURLsPendingImport.clear();
204 m_pageURLsInterestedInIcons.clear();
205 m_iconsPendingReading.clear();
206 m_loadersPendingDecision.clear();
210 m_removeIconsRequested = true;
214 Image* IconDatabase::synchronousIconForPageURL(const String& pageURLOriginal, const IntSize& size)
216 ASSERT_NOT_SYNC_THREAD();
218 // pageURLOriginal cannot be stored without being deep copied first.
219 // We should go our of our way to only copy it if we have to store it
221 if (!isOpen() || !documentCanHaveIcon(pageURLOriginal))
224 MutexLocker locker(m_urlAndIconLock);
226 String pageURLCopy; // Creates a null string for easy testing
228 PageURLRecord* pageRecord = m_pageURLToRecordMap.get(pageURLOriginal);
230 pageURLCopy = pageURLOriginal.crossThreadString();
231 pageRecord = getOrCreatePageURLRecord(pageURLCopy);
234 // If pageRecord is NULL, one of two things is true -
235 // 1 - The initial url import is incomplete and this pageURL was marked to be notified once it is complete if an iconURL exists
236 // 2 - The initial url import IS complete and this pageURL has no icon
238 MutexLocker locker(m_pendingReadingLock);
240 // Import is ongoing, there might be an icon. In this case, register to be notified when the icon comes in
241 // If we ever reach this condition, we know we've already made the pageURL copy
242 if (!m_iconURLImportComplete)
243 m_pageURLsInterestedInIcons.add(pageURLCopy);
248 IconRecord* iconRecord = pageRecord->iconRecord();
250 // If the initial URL import isn't complete, it's possible to have a PageURL record without an associated icon
251 // In this case, the pageURL is already in the set to alert the client when the iconURL mapping is complete so
252 // we can just bail now
253 if (!m_iconURLImportComplete && !iconRecord)
256 // The only way we should *not* have an icon record is if this pageURL is retained but has no icon yet - make sure of that
257 ASSERT(iconRecord || m_retainedPageURLs.contains(pageURLOriginal));
262 // If it's a new IconRecord object that doesn't have its imageData set yet,
263 // mark it to be read by the background thread
264 if (iconRecord->imageDataStatus() == ImageDataStatusUnknown) {
265 if (pageURLCopy.isNull())
266 pageURLCopy = pageURLOriginal.crossThreadString();
268 MutexLocker locker(m_pendingReadingLock);
269 m_pageURLsInterestedInIcons.add(pageURLCopy);
270 m_iconsPendingReading.add(iconRecord);
275 // If the size parameter was (0, 0) that means the caller of this method just wanted the read from disk to be kicked off
276 // and isn't actually interested in the image return value
277 if (size == IntSize(0, 0))
280 // PARANOID DISCUSSION: This method makes some assumptions. It returns a WebCore::image which the icon database might dispose of at anytime in the future,
281 // and Images aren't ref counted. So there is no way for the client to guarantee continued existence of the image.
282 // This has *always* been the case, but in practice clients would always create some other platform specific representation of the image
283 // and drop the raw Image*. On Mac an NSImage, and on windows drawing into an HBITMAP.
284 // The async aspect adds a huge question - what if the image is deleted before the platform specific API has a chance to create its own
285 // representation out of it?
286 // If an image is read in from the icondatabase, we do *not* overwrite any image data that exists in the in-memory cache.
287 // This is because we make the assumption that anything in memory is newer than whatever is in the database.
288 // So the only time the data will be set from the second thread is when it is INITIALLY being read in from the database, but we would never
289 // delete the image on the secondary thread if the image already exists.
290 return iconRecord->image(size);
293 void IconDatabase::readIconForPageURLFromDisk(const String& pageURL)
295 // The effect of asking for an Icon for a pageURL automatically queues it to be read from disk
296 // if it hasn't already been set in memory. The special IntSize (0, 0) is a special way of telling
297 // that method "I don't care about the actual Image, i just want you to make sure you're getting it from disk.
298 synchronousIconForPageURL(pageURL, IntSize(0, 0));
301 String IconDatabase::synchronousIconURLForPageURL(const String& pageURLOriginal)
303 ASSERT_NOT_SYNC_THREAD();
305 // Cannot do anything with pageURLOriginal that would end up storing it without deep copying first
306 // Also, in the case we have a real answer for the caller, we must deep copy that as well
308 if (!isOpen() || !documentCanHaveIcon(pageURLOriginal))
311 MutexLocker locker(m_urlAndIconLock);
313 PageURLRecord* pageRecord = m_pageURLToRecordMap.get(pageURLOriginal);
315 pageRecord = getOrCreatePageURLRecord(pageURLOriginal.crossThreadString());
317 // If pageRecord is NULL, one of two things is true -
318 // 1 - The initial url import is incomplete and this pageURL has already been marked to be notified once it is complete if an iconURL exists
319 // 2 - The initial url import IS complete and this pageURL has no icon
323 // Possible the pageRecord is around because it's a retained pageURL with no iconURL, so we have to check
324 return pageRecord->iconRecord() ? pageRecord->iconRecord()->iconURL().threadsafeCopy() : String();
327 #ifdef CAN_THEME_URL_ICON
328 static inline void loadDefaultIconRecord(IconRecord* defaultIconRecord)
330 defaultIconRecord->loadImageFromResource("urlIcon");
333 static inline void loadDefaultIconRecord(IconRecord* defaultIconRecord)
335 static const unsigned char defaultIconData[] = { 0x4D, 0x4D, 0x00, 0x2A, 0x00, 0x00, 0x03, 0x32, 0x80, 0x00, 0x20, 0x50, 0x38, 0x24, 0x16, 0x0D, 0x07, 0x84, 0x42, 0x61, 0x50, 0xB8,
336 0x64, 0x08, 0x18, 0x0D, 0x0A, 0x0B, 0x84, 0xA2, 0xA1, 0xE2, 0x08, 0x5E, 0x39, 0x28, 0xAF, 0x48, 0x24, 0xD3, 0x53, 0x9A, 0x37, 0x1D, 0x18, 0x0E, 0x8A, 0x4B, 0xD1, 0x38,
337 0xB0, 0x7C, 0x82, 0x07, 0x03, 0x82, 0xA2, 0xE8, 0x6C, 0x2C, 0x03, 0x2F, 0x02, 0x82, 0x41, 0xA1, 0xE2, 0xF8, 0xC8, 0x84, 0x68, 0x6D, 0x1C, 0x11, 0x0A, 0xB7, 0xFA, 0x91,
338 0x6E, 0xD1, 0x7F, 0xAF, 0x9A, 0x4E, 0x87, 0xFB, 0x19, 0xB0, 0xEA, 0x7F, 0xA4, 0x95, 0x8C, 0xB7, 0xF9, 0xA9, 0x0A, 0xA9, 0x7F, 0x8C, 0x88, 0x66, 0x96, 0xD4, 0xCA, 0x69,
339 0x2F, 0x00, 0x81, 0x65, 0xB0, 0x29, 0x90, 0x7C, 0xBA, 0x2B, 0x21, 0x1E, 0x5C, 0xE6, 0xB4, 0xBD, 0x31, 0xB6, 0xE7, 0x7A, 0xBF, 0xDD, 0x6F, 0x37, 0xD3, 0xFD, 0xD8, 0xF2,
340 0xB6, 0xDB, 0xED, 0xAC, 0xF7, 0x03, 0xC5, 0xFE, 0x77, 0x53, 0xB6, 0x1F, 0xE6, 0x24, 0x8B, 0x1D, 0xFE, 0x26, 0x20, 0x9E, 0x1C, 0xE0, 0x80, 0x65, 0x7A, 0x18, 0x02, 0x01,
341 0x82, 0xC5, 0xA0, 0xC0, 0xF1, 0x89, 0xBA, 0x23, 0x30, 0xAD, 0x1F, 0xE7, 0xE5, 0x5B, 0x6D, 0xFE, 0xE7, 0x78, 0x3E, 0x1F, 0xEE, 0x97, 0x8B, 0xE7, 0x37, 0x9D, 0xCF, 0xE7,
342 0x92, 0x8B, 0x87, 0x0B, 0xFC, 0xA0, 0x8E, 0x68, 0x3F, 0xC6, 0x27, 0xA6, 0x33, 0xFC, 0x36, 0x5B, 0x59, 0x3F, 0xC1, 0x02, 0x63, 0x3B, 0x74, 0x00, 0x03, 0x07, 0x0B, 0x61,
343 0x00, 0x20, 0x60, 0xC9, 0x08, 0x00, 0x1C, 0x25, 0x9F, 0xE0, 0x12, 0x8A, 0xD5, 0xFE, 0x6B, 0x4F, 0x35, 0x9F, 0xED, 0xD7, 0x4B, 0xD9, 0xFE, 0x8A, 0x59, 0xB8, 0x1F, 0xEC,
344 0x56, 0xD3, 0xC1, 0xFE, 0x63, 0x4D, 0xF2, 0x83, 0xC6, 0xB6, 0x1B, 0xFC, 0x34, 0x68, 0x61, 0x3F, 0xC1, 0xA6, 0x25, 0xEB, 0xFC, 0x06, 0x58, 0x5C, 0x3F, 0xC0, 0x03, 0xE4,
345 0xC3, 0xFC, 0x04, 0x0F, 0x1A, 0x6F, 0xE0, 0xE0, 0x20, 0xF9, 0x61, 0x7A, 0x02, 0x28, 0x2B, 0xBC, 0x46, 0x25, 0xF3, 0xFC, 0x66, 0x3D, 0x99, 0x27, 0xF9, 0x7E, 0x6B, 0x1D,
346 0xC7, 0xF9, 0x2C, 0x5E, 0x1C, 0x87, 0xF8, 0xC0, 0x4D, 0x9A, 0xE7, 0xF8, 0xDA, 0x51, 0xB2, 0xC1, 0x68, 0xF2, 0x64, 0x1F, 0xE1, 0x50, 0xED, 0x0A, 0x04, 0x23, 0x79, 0x8A,
347 0x7F, 0x82, 0xA3, 0x39, 0x80, 0x7F, 0x80, 0xC2, 0xB1, 0x5E, 0xF7, 0x04, 0x2F, 0xB2, 0x10, 0x02, 0x86, 0x63, 0xC9, 0xCC, 0x07, 0xBF, 0x87, 0xF8, 0x4A, 0x38, 0xAF, 0xC1,
348 0x88, 0xF8, 0x66, 0x1F, 0xE1, 0xD9, 0x08, 0xD4, 0x8F, 0x25, 0x5B, 0x4A, 0x49, 0x97, 0x87, 0x39, 0xFE, 0x25, 0x12, 0x10, 0x68, 0xAA, 0x4A, 0x2F, 0x42, 0x29, 0x12, 0x69,
349 0x9F, 0xE1, 0xC1, 0x00, 0x67, 0x1F, 0xE1, 0x58, 0xED, 0x00, 0x83, 0x23, 0x49, 0x82, 0x7F, 0x81, 0x21, 0xE0, 0xFC, 0x73, 0x21, 0x00, 0x50, 0x7D, 0x2B, 0x84, 0x03, 0x83,
350 0xC2, 0x1B, 0x90, 0x06, 0x69, 0xFE, 0x23, 0x91, 0xAE, 0x50, 0x9A, 0x49, 0x32, 0xC2, 0x89, 0x30, 0xE9, 0x0A, 0xC4, 0xD9, 0xC4, 0x7F, 0x94, 0xA6, 0x51, 0xDE, 0x7F, 0x9D,
351 0x07, 0x89, 0xF6, 0x7F, 0x91, 0x85, 0xCA, 0x88, 0x25, 0x11, 0xEE, 0x50, 0x7C, 0x43, 0x35, 0x21, 0x60, 0xF1, 0x0D, 0x82, 0x62, 0x39, 0x07, 0x2C, 0x20, 0xE0, 0x80, 0x72,
352 0x34, 0x17, 0xA1, 0x80, 0xEE, 0xF0, 0x89, 0x24, 0x74, 0x1A, 0x2C, 0x93, 0xB3, 0x78, 0xCC, 0x52, 0x9D, 0x6A, 0x69, 0x56, 0xBB, 0x0D, 0x85, 0x69, 0xE6, 0x7F, 0x9E, 0x27,
353 0xB9, 0xFD, 0x50, 0x54, 0x47, 0xF9, 0xCC, 0x78, 0x9F, 0x87, 0xF9, 0x98, 0x70, 0xB9, 0xC2, 0x91, 0x2C, 0x6D, 0x1F, 0xE1, 0xE1, 0x00, 0xBF, 0x02, 0xC1, 0xF5, 0x18, 0x84,
354 0x01, 0xE1, 0x48, 0x8C, 0x42, 0x07, 0x43, 0xC9, 0x76, 0x7F, 0x8B, 0x04, 0xE4, 0xDE, 0x35, 0x95, 0xAB, 0xB0, 0xF0, 0x5C, 0x55, 0x23, 0xF9, 0x7E, 0x7E, 0x9F, 0xE4, 0x0C,
355 0xA7, 0x55, 0x47, 0xC7, 0xF9, 0xE6, 0xCF, 0x1F, 0xE7, 0x93, 0x35, 0x52, 0x54, 0x63, 0x19, 0x46, 0x73, 0x1F, 0xE2, 0x61, 0x08, 0xF0, 0x82, 0xE1, 0x80, 0x92, 0xF9, 0x20,
356 0xC0, 0x28, 0x18, 0x0A, 0x05, 0xA1, 0xA2, 0xF8, 0x6E, 0xDB, 0x47, 0x49, 0xFE, 0x3E, 0x17, 0xB6, 0x61, 0x13, 0x1A, 0x29, 0x26, 0xA9, 0xFE, 0x7F, 0x92, 0x70, 0x69, 0xFE,
357 0x4C, 0x2F, 0x55, 0x01, 0xF1, 0x54, 0xD4, 0x35, 0x49, 0x4A, 0x69, 0x59, 0x83, 0x81, 0x58, 0x76, 0x9F, 0xE2, 0x20, 0xD6, 0x4C, 0x9B, 0xA0, 0x48, 0x1E, 0x0B, 0xB7, 0x48,
358 0x58, 0x26, 0x11, 0x06, 0x42, 0xE8, 0xA4, 0x40, 0x17, 0x27, 0x39, 0x00, 0x60, 0x2D, 0xA4, 0xC3, 0x2C, 0x7F, 0x94, 0x56, 0xE4, 0xE1, 0x77, 0x1F, 0xE5, 0xB9, 0xD7, 0x66,
359 0x1E, 0x07, 0xB3, 0x3C, 0x63, 0x1D, 0x35, 0x49, 0x0E, 0x63, 0x2D, 0xA2, 0xF1, 0x12, 0x60, 0x1C, 0xE0, 0xE0, 0x52, 0x1B, 0x8B, 0xAC, 0x38, 0x0E, 0x07, 0x03, 0x60, 0x28,
360 0x1C, 0x0E, 0x87, 0x00, 0xF0, 0x66, 0x27, 0x11, 0xA2, 0xC1, 0x02, 0x5A, 0x1C, 0xE4, 0x21, 0x83, 0x1F, 0x13, 0x86, 0xFA, 0xD2, 0x55, 0x1D, 0xD6, 0x61, 0xBC, 0x77, 0xD3,
361 0xE6, 0x91, 0xCB, 0x4C, 0x90, 0xA6, 0x25, 0xB8, 0x2F, 0x90, 0xC5, 0xA9, 0xCE, 0x12, 0x07, 0x02, 0x91, 0x1B, 0x9F, 0x68, 0x00, 0x16, 0x76, 0x0D, 0xA1, 0x00, 0x08, 0x06,
362 0x03, 0x81, 0xA0, 0x20, 0x1A, 0x0D, 0x06, 0x80, 0x30, 0x24, 0x12, 0x89, 0x20, 0x98, 0x4A, 0x1F, 0x0F, 0x21, 0xA0, 0x9E, 0x36, 0x16, 0xC2, 0x88, 0xE6, 0x48, 0x9B, 0x83,
363 0x31, 0x1C, 0x55, 0x1E, 0x43, 0x59, 0x1A, 0x56, 0x1E, 0x42, 0xF0, 0xFA, 0x4D, 0x1B, 0x9B, 0x08, 0xDC, 0x5B, 0x02, 0xA1, 0x30, 0x7E, 0x3C, 0xEE, 0x5B, 0xA6, 0xDD, 0xB8,
364 0x6D, 0x5B, 0x62, 0xB7, 0xCD, 0xF3, 0x9C, 0xEA, 0x04, 0x80, 0x80, 0x00, 0x00, 0x0E, 0x01, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x10, 0x00, 0x00, 0x01, 0x01,
365 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x10, 0x00, 0x00, 0x01, 0x02, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x03, 0xE0, 0x01, 0x03, 0x00, 0x03, 0x00, 0x00,
366 0x00, 0x01, 0x00, 0x05, 0x00, 0x00, 0x01, 0x06, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x00, 0x01, 0x11, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00,
367 0x00, 0x08, 0x01, 0x15, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x01, 0x16, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x02, 0x00, 0x01, 0x17,
368 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x03, 0x29, 0x01, 0x1A, 0x00, 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x03, 0xE8, 0x01, 0x1B, 0x00, 0x05, 0x00, 0x00,
369 0x00, 0x01, 0x00, 0x00, 0x03, 0xF0, 0x01, 0x1C, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x01, 0x28, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02,
370 0x00, 0x00, 0x01, 0x52, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x08, 0x00, 0x08, 0x00, 0x08, 0x00, 0x0A,
371 0xFC, 0x80, 0x00, 0x00, 0x27, 0x10, 0x00, 0x0A, 0xFC, 0x80, 0x00, 0x00, 0x27, 0x10 };
373 DEFINE_STATIC_LOCAL(RefPtr<SharedBuffer>, defaultIconBuffer, (SharedBuffer::create(defaultIconData, sizeof(defaultIconData))));
374 defaultIconRecord->setImageData(defaultIconBuffer);
378 Image* IconDatabase::defaultIcon(const IntSize& size)
380 ASSERT_NOT_SYNC_THREAD();
383 if (!m_defaultIconRecord) {
384 m_defaultIconRecord = IconRecord::create("urlIcon");
385 m_defaultIconRecord->setMutexForVerifier(m_urlAndIconLock);
386 loadDefaultIconRecord(m_defaultIconRecord.get());
389 return m_defaultIconRecord->image(size);
393 void IconDatabase::retainIconForPageURL(const String& pageURLOriginal)
395 ASSERT_NOT_SYNC_THREAD();
397 // Cannot do anything with pageURLOriginal that would end up storing it without deep copying first
399 if (!isEnabled() || !documentCanHaveIcon(pageURLOriginal))
402 MutexLocker locker(m_urlAndIconLock);
404 PageURLRecord* record = m_pageURLToRecordMap.get(pageURLOriginal);
409 pageURL = pageURLOriginal.crossThreadString();
411 record = new PageURLRecord(pageURL);
412 m_pageURLToRecordMap.set(pageURL, record);
415 if (!record->retain()) {
416 if (pageURL.isNull())
417 pageURL = pageURLOriginal.crossThreadString();
419 // This page just had its retain count bumped from 0 to 1 - Record that fact
420 m_retainedPageURLs.add(pageURL);
422 // If we read the iconURLs yet, we want to avoid any pageURL->iconURL lookups and the pageURLsPendingDeletion is moot,
423 // so we bail here and skip those steps
424 if (!m_iconURLImportComplete)
427 MutexLocker locker(m_pendingSyncLock);
428 // If this pageURL waiting to be sync'ed, update the sync record
429 // This saves us in the case where a page was ready to be deleted from the database but was just retained - so theres no need to delete it!
430 if (!m_privateBrowsingEnabled && m_pageURLsPendingSync.contains(pageURL)) {
431 LOG(IconDatabase, "Bringing %s back from the brink", pageURL.ascii().data());
432 m_pageURLsPendingSync.set(pageURL, record->snapshot());
437 void IconDatabase::releaseIconForPageURL(const String& pageURLOriginal)
439 ASSERT_NOT_SYNC_THREAD();
441 // Cannot do anything with pageURLOriginal that would end up storing it without deep copying first
443 if (!isEnabled() || !documentCanHaveIcon(pageURLOriginal))
446 MutexLocker locker(m_urlAndIconLock);
448 // Check if this pageURL is actually retained
449 if (!m_retainedPageURLs.contains(pageURLOriginal)) {
450 LOG_ERROR("Attempting to release icon for URL %s which is not retained", urlForLogging(pageURLOriginal).ascii().data());
454 // Get its retain count - if it's retained, we'd better have a PageURLRecord for it
455 PageURLRecord* pageRecord = m_pageURLToRecordMap.get(pageURLOriginal);
457 LOG(IconDatabase, "Releasing pageURL %s to a retain count of %i", urlForLogging(pageURLOriginal).ascii().data(), pageRecord->retainCount() - 1);
458 ASSERT(pageRecord->retainCount() > 0);
460 // If it still has a positive retain count, store the new count and bail
461 if (pageRecord->release())
464 // This pageRecord has now been fully released. Do the appropriate cleanup
465 LOG(IconDatabase, "No more retainers for PageURL %s", urlForLogging(pageURLOriginal).ascii().data());
466 m_pageURLToRecordMap.remove(pageURLOriginal);
467 m_retainedPageURLs.remove(pageURLOriginal);
469 // Grab the iconRecord for later use (and do a sanity check on it for kicks)
470 IconRecord* iconRecord = pageRecord->iconRecord();
472 ASSERT(!iconRecord || (iconRecord && m_iconURLToRecordMap.get(iconRecord->iconURL()) == iconRecord));
475 MutexLocker locker(m_pendingReadingLock);
477 // Since this pageURL is going away, there's no reason anyone would ever be interested in its read results
478 if (!m_iconURLImportComplete)
479 m_pageURLsPendingImport.remove(pageURLOriginal);
480 m_pageURLsInterestedInIcons.remove(pageURLOriginal);
482 // If this icon is down to it's last retainer, we don't care about reading it in from disk anymore
483 if (iconRecord && iconRecord->hasOneRef()) {
484 m_iconURLToRecordMap.remove(iconRecord->iconURL());
485 m_iconsPendingReading.remove(iconRecord);
489 // Mark stuff for deletion from the database only if we're not in private browsing
490 if (!m_privateBrowsingEnabled) {
491 MutexLocker locker(m_pendingSyncLock);
492 m_pageURLsPendingSync.set(pageURLOriginal.crossThreadString(), pageRecord->snapshot(true));
494 // If this page is the last page to refer to a particular IconRecord, that IconRecord needs to
495 // be marked for deletion
496 if (iconRecord && iconRecord->hasOneRef())
497 m_iconsPendingSync.set(iconRecord->iconURL(), iconRecord->snapshot(true));
503 scheduleOrDeferSyncTimer();
506 void IconDatabase::setIconDataForIconURL(PassRefPtr<SharedBuffer> dataOriginal, const String& iconURLOriginal)
508 ASSERT_NOT_SYNC_THREAD();
510 // Cannot do anything with dataOriginal or iconURLOriginal that would end up storing them without deep copying first
512 if (!isOpen() || iconURLOriginal.isEmpty())
515 RefPtr<SharedBuffer> data = dataOriginal ? dataOriginal->copy() : PassRefPtr<SharedBuffer>(0);
517 data->setMutexForVerifier(m_urlAndIconLock);
518 String iconURL = iconURLOriginal.crossThreadString();
520 Vector<String> pageURLs;
522 MutexLocker locker(m_urlAndIconLock);
524 // If this icon was pending a read, remove it from that set because this new data should override what is on disk
525 RefPtr<IconRecord> icon = m_iconURLToRecordMap.get(iconURL);
527 MutexLocker locker(m_pendingReadingLock);
528 m_iconsPendingReading.remove(icon.get());
530 icon = getOrCreateIconRecord(iconURL);
532 // Update the data and set the time stamp
533 icon->setImageData(data.release());
534 icon->setTimestamp((int)currentTime());
536 // Copy the current retaining pageURLs - if any - to notify them of the change
537 pageURLs.appendRange(icon->retainingPageURLs().begin(), icon->retainingPageURLs().end());
539 // Mark the IconRecord as requiring an update to the database only if private browsing is disabled
540 if (!m_privateBrowsingEnabled) {
541 MutexLocker locker(m_pendingSyncLock);
542 m_iconsPendingSync.set(iconURL, icon->snapshot());
545 if (icon->hasOneRef()) {
546 ASSERT(icon->retainingPageURLs().isEmpty());
547 LOG(IconDatabase, "Icon for icon url %s is about to be destroyed - removing mapping for it", urlForLogging(icon->iconURL()).ascii().data());
548 m_iconURLToRecordMap.remove(icon->iconURL());
552 // Send notification out regarding all PageURLs that retain this icon
553 // But not if we're on the sync thread because that implies this mapping
554 // comes from the initial import which we don't want notifications for
555 if (!IS_ICON_SYNC_THREAD()) {
556 // Start the timer to commit this change - or further delay the timer if it was already started
557 scheduleOrDeferSyncTimer();
559 // Informal testing shows that draining the autorelease pool every 25 iterations is about as low as we can go
560 // before performance starts to drop off, but we don't want to increase this number because then accumulated memory usage will go up
561 AutodrainedPool pool(25);
563 for (unsigned i = 0; i < pageURLs.size(); ++i) {
564 LOG(IconDatabase, "Dispatching notification that retaining pageURL %s has a new icon", urlForLogging(pageURLs[i]).ascii().data());
565 m_client->didChangeIconForPageURL(pageURLs[i]);
572 void IconDatabase::setIconURLForPageURL(const String& iconURLOriginal, const String& pageURLOriginal)
574 ASSERT_NOT_SYNC_THREAD();
576 // Cannot do anything with iconURLOriginal or pageURLOriginal that would end up storing them without deep copying first
578 ASSERT(!iconURLOriginal.isEmpty());
580 if (!isOpen() || !documentCanHaveIcon(pageURLOriginal))
583 String iconURL, pageURL;
586 MutexLocker locker(m_urlAndIconLock);
588 PageURLRecord* pageRecord = m_pageURLToRecordMap.get(pageURLOriginal);
590 // If the urls already map to each other, bail.
591 // This happens surprisingly often, and seems to cream iBench performance
592 if (pageRecord && pageRecord->iconRecord() && pageRecord->iconRecord()->iconURL() == iconURLOriginal)
595 pageURL = pageURLOriginal.crossThreadString();
596 iconURL = iconURLOriginal.crossThreadString();
599 pageRecord = new PageURLRecord(pageURL);
600 m_pageURLToRecordMap.set(pageURL, pageRecord);
603 RefPtr<IconRecord> iconRecord = pageRecord->iconRecord();
605 // Otherwise, set the new icon record for this page
606 pageRecord->setIconRecord(getOrCreateIconRecord(iconURL));
608 // If the current icon has only a single ref left, it is about to get wiped out.
609 // Remove it from the in-memory records and don't bother reading it in from disk anymore
610 if (iconRecord && iconRecord->hasOneRef()) {
611 ASSERT(iconRecord->retainingPageURLs().size() == 0);
612 LOG(IconDatabase, "Icon for icon url %s is about to be destroyed - removing mapping for it", urlForLogging(iconRecord->iconURL()).ascii().data());
613 m_iconURLToRecordMap.remove(iconRecord->iconURL());
614 MutexLocker locker(m_pendingReadingLock);
615 m_iconsPendingReading.remove(iconRecord.get());
618 // And mark this mapping to be added to the database
619 if (!m_privateBrowsingEnabled) {
620 MutexLocker locker(m_pendingSyncLock);
621 m_pageURLsPendingSync.set(pageURL, pageRecord->snapshot());
623 // If the icon is on its last ref, mark it for deletion
624 if (iconRecord && iconRecord->hasOneRef())
625 m_iconsPendingSync.set(iconRecord->iconURL(), iconRecord->snapshot(true));
629 // Since this mapping is new, send the notification out - but not if we're on the sync thread because that implies this mapping
630 // comes from the initial import which we don't want notifications for
631 if (!IS_ICON_SYNC_THREAD()) {
632 // Start the timer to commit this change - or further delay the timer if it was already started
633 scheduleOrDeferSyncTimer();
635 LOG(IconDatabase, "Dispatching notification that we changed an icon mapping for url %s", urlForLogging(pageURL).ascii().data());
636 AutodrainedPool pool;
637 m_client->didChangeIconForPageURL(pageURL);
641 IconLoadDecision IconDatabase::synchronousLoadDecisionForIconURL(const String& iconURL, DocumentLoader* notificationDocumentLoader)
643 ASSERT_NOT_SYNC_THREAD();
645 if (!isOpen() || iconURL.isEmpty())
648 // If we have a IconRecord, it should also have its timeStamp marked because there is only two times when we create the IconRecord:
649 // 1 - When we read the icon urls from disk, getting the timeStamp at the same time
650 // 2 - When we get a new icon from the loader, in which case the timestamp is set at that time
652 MutexLocker locker(m_urlAndIconLock);
653 if (IconRecord* icon = m_iconURLToRecordMap.get(iconURL)) {
654 LOG(IconDatabase, "Found expiration time on a present icon based on existing IconRecord");
655 return (int)currentTime() - icon->getTimestamp() > iconExpirationTime ? IconLoadYes : IconLoadNo;
659 // If we don't have a record for it, but we *have* imported all iconURLs from disk, then we should load it now
660 MutexLocker readingLocker(m_pendingReadingLock);
661 if (m_iconURLImportComplete)
664 // Otherwise - since we refuse to perform I/O on the main thread to find out for sure - we return the answer that says
665 // "You might be asked to load this later, so flag that"
666 LOG(IconDatabase, "Don't know if we should load %s or not - adding %p to the set of document loaders waiting on a decision", iconURL.ascii().data(), notificationDocumentLoader);
667 if (notificationDocumentLoader)
668 m_loadersPendingDecision.add(notificationDocumentLoader);
670 return IconLoadUnknown;
673 bool IconDatabase::synchronousIconDataKnownForIconURL(const String& iconURL)
675 ASSERT_NOT_SYNC_THREAD();
677 MutexLocker locker(m_urlAndIconLock);
678 if (IconRecord* icon = m_iconURLToRecordMap.get(iconURL))
679 return icon->imageDataStatus() != ImageDataStatusUnknown;
684 void IconDatabase::setEnabled(bool enabled)
686 ASSERT_NOT_SYNC_THREAD();
688 if (!enabled && isOpen())
690 m_isEnabled = enabled;
693 bool IconDatabase::isEnabled() const
695 ASSERT_NOT_SYNC_THREAD();
700 void IconDatabase::setPrivateBrowsingEnabled(bool flag)
702 m_privateBrowsingEnabled = flag;
705 bool IconDatabase::isPrivateBrowsingEnabled() const
707 return m_privateBrowsingEnabled;
710 void IconDatabase::delayDatabaseCleanup()
712 ++databaseCleanupCounter;
713 if (databaseCleanupCounter == 1)
714 LOG(IconDatabase, "Database cleanup is now DISABLED");
717 void IconDatabase::allowDatabaseCleanup()
719 if (--databaseCleanupCounter < 0)
720 databaseCleanupCounter = 0;
721 if (databaseCleanupCounter == 0)
722 LOG(IconDatabase, "Database cleanup is now ENABLED");
725 void IconDatabase::checkIntegrityBeforeOpening()
727 checkIntegrityOnOpen = true;
730 size_t IconDatabase::pageURLMappingCount()
732 MutexLocker locker(m_urlAndIconLock);
733 return m_pageURLToRecordMap.size();
736 size_t IconDatabase::retainedPageURLCount()
738 MutexLocker locker(m_urlAndIconLock);
739 return m_retainedPageURLs.size();
742 size_t IconDatabase::iconRecordCount()
744 MutexLocker locker(m_urlAndIconLock);
745 return m_iconURLToRecordMap.size();
748 size_t IconDatabase::iconRecordCountWithData()
750 MutexLocker locker(m_urlAndIconLock);
753 HashMap<String, IconRecord*>::iterator i = m_iconURLToRecordMap.begin();
754 HashMap<String, IconRecord*>::iterator end = m_iconURLToRecordMap.end();
756 for (; i != end; ++i)
757 result += ((*i).second->imageDataStatus() == ImageDataStatusPresent);
762 IconDatabase::IconDatabase()
763 : m_syncTimer(this, &IconDatabase::syncTimerFired)
764 , m_syncThreadRunning(false)
766 , m_privateBrowsingEnabled(false)
767 , m_threadTerminationRequested(false)
768 , m_removeIconsRequested(false)
769 , m_iconURLImportComplete(false)
770 , m_disabledSuddenTerminationForSyncThread(false)
771 , m_initialPruningComplete(false)
772 , m_client(defaultClient())
774 , m_isImportedSet(false)
776 LOG(IconDatabase, "Creating IconDatabase %p", this);
777 ASSERT(isMainThread());
780 IconDatabase::~IconDatabase()
782 ASSERT_NOT_REACHED();
785 void IconDatabase::notifyPendingLoadDecisionsOnMainThread(void* context)
787 static_cast<IconDatabase*>(context)->notifyPendingLoadDecisions();
790 void IconDatabase::notifyPendingLoadDecisions()
792 ASSERT_NOT_SYNC_THREAD();
794 // This method should only be called upon completion of the initial url import from the database
795 ASSERT(m_iconURLImportComplete);
796 LOG(IconDatabase, "Notifying all DocumentLoaders that were waiting on a load decision for thier icons");
798 HashSet<RefPtr<DocumentLoader> >::iterator i = m_loadersPendingDecision.begin();
799 HashSet<RefPtr<DocumentLoader> >::iterator end = m_loadersPendingDecision.end();
801 for (; i != end; ++i)
802 if ((*i)->refCount() > 1)
803 (*i)->iconLoadDecisionAvailable();
805 m_loadersPendingDecision.clear();
808 void IconDatabase::wakeSyncThread()
810 MutexLocker locker(m_syncLock);
812 if (!m_disabledSuddenTerminationForSyncThread) {
813 m_disabledSuddenTerminationForSyncThread = true;
814 // The following is balanced by the call to enableSuddenTermination in the
815 // syncThreadMainLoop function.
816 // FIXME: It would be better to only disable sudden termination if we have
817 // something to write, not just if we have something to read.
818 disableSuddenTermination();
821 m_syncCondition.signal();
824 void IconDatabase::scheduleOrDeferSyncTimer()
826 ASSERT_NOT_SYNC_THREAD();
828 if (!m_syncTimer.isActive()) {
829 // The following is balanced by the call to enableSuddenTermination in the
830 // syncTimerFired function.
831 disableSuddenTermination();
834 m_syncTimer.startOneShot(updateTimerDelay);
837 void IconDatabase::syncTimerFired(Timer<IconDatabase>*)
839 ASSERT_NOT_SYNC_THREAD();
842 // The following is balanced by the call to disableSuddenTermination in the
843 // scheduleOrDeferSyncTimer function.
844 enableSuddenTermination();
847 // ******************
848 // *** Any Thread ***
849 // ******************
851 bool IconDatabase::isOpen() const
853 MutexLocker locker(m_syncLock);
854 return m_syncDB.isOpen();
857 String IconDatabase::databasePath() const
859 MutexLocker locker(m_syncLock);
860 return m_completeDatabasePath.threadsafeCopy();
863 String IconDatabase::defaultDatabaseFilename()
865 DEFINE_STATIC_LOCAL(String, defaultDatabaseFilename, ("WebpageIcons.db"));
866 return defaultDatabaseFilename.threadsafeCopy();
869 // Unlike getOrCreatePageURLRecord(), getOrCreateIconRecord() does not mark the icon as "interested in import"
870 PassRefPtr<IconRecord> IconDatabase::getOrCreateIconRecord(const String& iconURL)
872 // Clients of getOrCreateIconRecord() are required to acquire the m_urlAndIconLock before calling this method
873 ASSERT(!m_urlAndIconLock.tryLock());
875 if (IconRecord* icon = m_iconURLToRecordMap.get(iconURL))
878 RefPtr<IconRecord> newIcon = IconRecord::create(iconURL);
879 newIcon->setMutexForVerifier(m_urlAndIconLock);
880 m_iconURLToRecordMap.set(iconURL, newIcon.get());
882 return newIcon.release();
885 // This method retrieves the existing PageURLRecord, or creates a new one and marks it as "interested in the import" for later notification
886 PageURLRecord* IconDatabase::getOrCreatePageURLRecord(const String& pageURL)
888 // Clients of getOrCreatePageURLRecord() are required to acquire the m_urlAndIconLock before calling this method
889 ASSERT(!m_urlAndIconLock.tryLock());
891 if (!documentCanHaveIcon(pageURL))
894 PageURLRecord* pageRecord = m_pageURLToRecordMap.get(pageURL);
896 MutexLocker locker(m_pendingReadingLock);
897 if (!m_iconURLImportComplete) {
898 // If the initial import of all URLs hasn't completed and we have no page record, we assume we *might* know about this later and create a record for it
900 LOG(IconDatabase, "Creating new PageURLRecord for pageURL %s", urlForLogging(pageURL).ascii().data());
901 pageRecord = new PageURLRecord(pageURL);
902 m_pageURLToRecordMap.set(pageURL, pageRecord);
905 // If the pageRecord for this page does not have an iconRecord attached to it, then it is a new pageRecord still awaiting the initial import
906 // Mark the URL as "interested in the result of the import" then bail
907 if (!pageRecord->iconRecord()) {
908 m_pageURLsPendingImport.add(pageURL);
913 // We've done the initial import of all URLs known in the database. If this record doesn't exist now, it never will
918 // ************************
919 // *** Sync Thread Only ***
920 // ************************
922 void IconDatabase::importIconURLForPageURL(const String& iconURL, const String& pageURL)
924 ASSERT_ICON_SYNC_THREAD();
926 // This function is only for setting actual existing url mappings so assert that neither of these URLs are empty
927 ASSERT(!iconURL.isEmpty());
928 ASSERT(!pageURL.isEmpty());
929 ASSERT(documentCanHaveIcon(pageURL));
931 setIconURLForPageURLInSQLDatabase(iconURL, pageURL);
934 void IconDatabase::importIconDataForIconURL(PassRefPtr<SharedBuffer> data, const String& iconURL)
936 ASSERT_ICON_SYNC_THREAD();
938 ASSERT(!iconURL.isEmpty());
940 writeIconSnapshotToSQLDatabase(IconSnapshot(iconURL, (int)currentTime(), data.get()));
943 bool IconDatabase::shouldStopThreadActivity() const
945 ASSERT_ICON_SYNC_THREAD();
947 return m_threadTerminationRequested || m_removeIconsRequested;
950 void* IconDatabase::iconDatabaseSyncThreadStart(void* vIconDatabase)
952 IconDatabase* iconDB = static_cast<IconDatabase*>(vIconDatabase);
954 return iconDB->iconDatabaseSyncThread();
957 void* IconDatabase::iconDatabaseSyncThread()
959 // The call to create this thread might not complete before the thread actually starts, so we might fail this ASSERT_ICON_SYNC_THREAD() because the pointer
960 // to our thread structure hasn't been filled in yet.
961 // To fix this, the main thread acquires this lock before creating us, then releases the lock after creation is complete. A quick lock/unlock cycle here will
962 // prevent us from running before that call completes
966 ASSERT_ICON_SYNC_THREAD();
968 LOG(IconDatabase, "(THREAD) IconDatabase sync thread started");
971 double startTime = currentTime();
974 // Need to create the database path if it doesn't already exist
975 makeAllDirectories(m_databaseDirectory);
977 // Existence of a journal file is evidence of a previous crash/force quit and automatically qualifies
978 // us to do an integrity check
979 String journalFilename = m_completeDatabasePath + "-journal";
980 if (!checkIntegrityOnOpen) {
981 AutodrainedPool pool;
982 checkIntegrityOnOpen = fileExists(journalFilename);
986 MutexLocker locker(m_syncLock);
987 if (!m_syncDB.open(m_completeDatabasePath)) {
988 LOG_ERROR("Unable to open icon database at path %s - %s", m_completeDatabasePath.ascii().data(), m_syncDB.lastErrorMsg());
993 if (shouldStopThreadActivity())
994 return syncThreadMainLoop();
997 double timeStamp = currentTime();
998 LOG(IconDatabase, "(THREAD) Open took %.4f seconds", timeStamp - startTime);
1001 performOpenInitialization();
1002 if (shouldStopThreadActivity())
1003 return syncThreadMainLoop();
1006 double newStamp = currentTime();
1007 LOG(IconDatabase, "(THREAD) performOpenInitialization() took %.4f seconds, now %.4f seconds from thread start", newStamp - timeStamp, newStamp - startTime);
1008 timeStamp = newStamp;
1012 LOG(IconDatabase, "(THREAD) Performing Safari2 import procedure");
1013 SQLiteTransaction importTransaction(m_syncDB);
1014 importTransaction.begin();
1016 // Commit the transaction only if the import completes (the import should be atomic)
1017 if (m_client->performImport()) {
1019 importTransaction.commit();
1021 LOG(IconDatabase, "(THREAD) Safari 2 import was cancelled");
1022 importTransaction.rollback();
1025 if (shouldStopThreadActivity())
1026 return syncThreadMainLoop();
1029 newStamp = currentTime();
1030 LOG(IconDatabase, "(THREAD) performImport() took %.4f seconds, now %.4f seconds from thread start", newStamp - timeStamp, newStamp - startTime);
1031 timeStamp = newStamp;
1035 // Uncomment the following line to simulate a long lasting URL import (*HUGE* icon databases, or network home directories)
1036 // while (currentTime() - timeStamp < 10);
1038 // Read in URL mappings from the database
1039 LOG(IconDatabase, "(THREAD) Starting iconURL import");
1042 if (shouldStopThreadActivity())
1043 return syncThreadMainLoop();
1046 newStamp = currentTime();
1047 LOG(IconDatabase, "(THREAD) performURLImport() took %.4f seconds. Entering main loop %.4f seconds from thread start", newStamp - timeStamp, newStamp - startTime);
1050 LOG(IconDatabase, "(THREAD) Beginning sync");
1051 return syncThreadMainLoop();
1054 static int databaseVersionNumber(SQLiteDatabase& db)
1056 return SQLiteStatement(db, "SELECT value FROM IconDatabaseInfo WHERE key = 'Version';").getColumnInt(0);
1059 static bool isValidDatabase(SQLiteDatabase& db)
1061 // These four tables should always exist in a valid db
1062 if (!db.tableExists("IconInfo") || !db.tableExists("IconData") || !db.tableExists("PageURL") || !db.tableExists("IconDatabaseInfo"))
1065 if (databaseVersionNumber(db) < currentDatabaseVersion) {
1066 LOG(IconDatabase, "DB version is not found or below expected valid version");
1073 static void createDatabaseTables(SQLiteDatabase& db)
1075 if (!db.executeCommand("CREATE TABLE PageURL (url TEXT NOT NULL ON CONFLICT FAIL UNIQUE ON CONFLICT REPLACE,iconID INTEGER NOT NULL ON CONFLICT FAIL);")) {
1076 LOG_ERROR("Could not create PageURL table in database (%i) - %s", db.lastError(), db.lastErrorMsg());
1080 if (!db.executeCommand("CREATE INDEX PageURLIndex ON PageURL (url);")) {
1081 LOG_ERROR("Could not create PageURL index in database (%i) - %s", db.lastError(), db.lastErrorMsg());
1085 if (!db.executeCommand("CREATE TABLE IconInfo (iconID INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE ON CONFLICT REPLACE, url TEXT NOT NULL ON CONFLICT FAIL UNIQUE ON CONFLICT FAIL, stamp INTEGER);")) {
1086 LOG_ERROR("Could not create IconInfo table in database (%i) - %s", db.lastError(), db.lastErrorMsg());
1090 if (!db.executeCommand("CREATE INDEX IconInfoIndex ON IconInfo (url, iconID);")) {
1091 LOG_ERROR("Could not create PageURL index in database (%i) - %s", db.lastError(), db.lastErrorMsg());
1095 if (!db.executeCommand("CREATE TABLE IconData (iconID INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE ON CONFLICT REPLACE, data BLOB);")) {
1096 LOG_ERROR("Could not create IconData table in database (%i) - %s", db.lastError(), db.lastErrorMsg());
1100 if (!db.executeCommand("CREATE INDEX IconDataIndex ON IconData (iconID);")) {
1101 LOG_ERROR("Could not create PageURL index in database (%i) - %s", db.lastError(), db.lastErrorMsg());
1105 if (!db.executeCommand("CREATE TABLE IconDatabaseInfo (key TEXT NOT NULL ON CONFLICT FAIL UNIQUE ON CONFLICT REPLACE,value TEXT NOT NULL ON CONFLICT FAIL);")) {
1106 LOG_ERROR("Could not create IconDatabaseInfo table in database (%i) - %s", db.lastError(), db.lastErrorMsg());
1110 if (!db.executeCommand(String("INSERT INTO IconDatabaseInfo VALUES ('Version', ") + String::number(currentDatabaseVersion) + ");")) {
1111 LOG_ERROR("Could not insert icon database version into IconDatabaseInfo table (%i) - %s", db.lastError(), db.lastErrorMsg());
1117 void IconDatabase::performOpenInitialization()
1119 ASSERT_ICON_SYNC_THREAD();
1124 if (checkIntegrityOnOpen) {
1125 checkIntegrityOnOpen = false;
1126 if (!checkIntegrity()) {
1127 LOG(IconDatabase, "Integrity check was bad - dumping IconDatabase");
1132 MutexLocker locker(m_syncLock);
1133 // Should've been consumed by SQLite, delete just to make sure we don't see it again in the future;
1134 deleteFile(m_completeDatabasePath + "-journal");
1135 deleteFile(m_completeDatabasePath);
1138 // Reopen the write database, creating it from scratch
1139 if (!m_syncDB.open(m_completeDatabasePath)) {
1140 LOG_ERROR("Unable to open icon database at path %s - %s", m_completeDatabasePath.ascii().data(), m_syncDB.lastErrorMsg());
1146 int version = databaseVersionNumber(m_syncDB);
1148 if (version > currentDatabaseVersion) {
1149 LOG(IconDatabase, "Database version number %i is greater than our current version number %i - closing the database to prevent overwriting newer versions", version, currentDatabaseVersion);
1151 m_threadTerminationRequested = true;
1155 if (!isValidDatabase(m_syncDB)) {
1156 LOG(IconDatabase, "%s is missing or in an invalid state - reconstructing", m_completeDatabasePath.ascii().data());
1157 m_syncDB.clearAllTables();
1158 createDatabaseTables(m_syncDB);
1161 // Reduce sqlite RAM cache size from default 2000 pages (~1.5kB per page). 3MB of cache for icon database is overkill
1162 if (!SQLiteStatement(m_syncDB, "PRAGMA cache_size = 200;").executeCommand())
1163 LOG_ERROR("SQLite database could not set cache_size");
1165 // Tell backup software (i.e., Time Machine) to never back up the icon database, because
1166 // it's a large file that changes frequently, thus using a lot of backup disk space, and
1167 // it's unlikely that many users would be upset about it not being backed up. We could
1168 // make this configurable on a per-client basis some day if some clients don't want this.
1169 if (canExcludeFromBackup() && !wasExcludedFromBackup() && excludeFromBackup(m_completeDatabasePath))
1170 setWasExcludedFromBackup();
1173 bool IconDatabase::checkIntegrity()
1175 ASSERT_ICON_SYNC_THREAD();
1177 SQLiteStatement integrity(m_syncDB, "PRAGMA integrity_check;");
1178 if (integrity.prepare() != SQLResultOk) {
1179 LOG_ERROR("checkIntegrity failed to execute");
1183 int resultCode = integrity.step();
1184 if (resultCode == SQLResultOk)
1187 if (resultCode != SQLResultRow)
1190 int columns = integrity.columnCount();
1192 LOG_ERROR("Received %i columns performing integrity check, should be 1", columns);
1196 String resultText = integrity.getColumnText(0);
1198 // A successful, no-error integrity check will be "ok" - all other strings imply failure
1199 if (resultText == "ok")
1202 LOG_ERROR("Icon database integrity check failed - \n%s", resultText.ascii().data());
1206 void IconDatabase::performURLImport()
1208 ASSERT_ICON_SYNC_THREAD();
1210 SQLiteStatement query(m_syncDB, "SELECT PageURL.url, IconInfo.url, IconInfo.stamp FROM PageURL INNER JOIN IconInfo ON PageURL.iconID=IconInfo.iconID;");
1212 if (query.prepare() != SQLResultOk) {
1213 LOG_ERROR("Unable to prepare icon url import query");
1217 // Informal testing shows that draining the autorelease pool every 25 iterations is about as low as we can go
1218 // before performance starts to drop off, but we don't want to increase this number because then accumulated memory usage will go up
1219 AutodrainedPool pool(25);
1221 int result = query.step();
1222 while (result == SQLResultRow) {
1223 String pageURL = query.getColumnText(0);
1224 String iconURL = query.getColumnText(1);
1227 MutexLocker locker(m_urlAndIconLock);
1229 PageURLRecord* pageRecord = m_pageURLToRecordMap.get(pageURL);
1231 // If the pageRecord doesn't exist in this map, then no one has retained this pageURL
1232 // If the s_databaseCleanupCounter count is non-zero, then we're not supposed to be pruning the database in any manner,
1233 // so go ahead and actually create a pageURLRecord for this url even though it's not retained.
1234 // If database cleanup *is* allowed, we don't want to bother pulling in a page url from disk that noone is actually interested
1235 // in - we'll prune it later instead!
1236 if (!pageRecord && databaseCleanupCounter && documentCanHaveIcon(pageURL)) {
1237 pageRecord = new PageURLRecord(pageURL);
1238 m_pageURLToRecordMap.set(pageURL, pageRecord);
1242 IconRecord* currentIcon = pageRecord->iconRecord();
1244 if (!currentIcon || currentIcon->iconURL() != iconURL) {
1245 pageRecord->setIconRecord(getOrCreateIconRecord(iconURL));
1246 currentIcon = pageRecord->iconRecord();
1249 // Regardless, the time stamp from disk still takes precedence. Until we read this icon from disk, we didn't think we'd seen it before
1250 // so we marked the timestamp as "now", but it's really much older
1251 currentIcon->setTimestamp(query.getColumnInt(2));
1255 // FIXME: Currently the WebKit API supports 1 type of notification that is sent whenever we get an Icon URL for a Page URL. We might want to re-purpose it to work for
1256 // getting the actually icon itself also (so each pageurl would get this notification twice) or we might want to add a second type of notification -
1257 // one for the URL and one for the Image itself
1258 // Note that WebIconDatabase is not neccessarily API so we might be able to make this change
1260 MutexLocker locker(m_pendingReadingLock);
1261 if (m_pageURLsPendingImport.contains(pageURL)) {
1262 dispatchDidImportIconURLForPageURLOnMainThread(pageURL);
1263 m_pageURLsPendingImport.remove(pageURL);
1269 // Stop the import at any time of the thread has been asked to shutdown
1270 if (shouldStopThreadActivity()) {
1271 LOG(IconDatabase, "IconDatabase asked to terminate during performURLImport()");
1275 result = query.step();
1278 if (result != SQLResultDone)
1279 LOG(IconDatabase, "Error reading page->icon url mappings from database");
1281 // Clear the m_pageURLsPendingImport set - either the page URLs ended up with an iconURL (that we'll notify about) or not,
1282 // but after m_iconURLImportComplete is set to true, we don't care about this set anymore
1283 Vector<String> urls;
1285 MutexLocker locker(m_pendingReadingLock);
1287 urls.appendRange(m_pageURLsPendingImport.begin(), m_pageURLsPendingImport.end());
1288 m_pageURLsPendingImport.clear();
1289 m_iconURLImportComplete = true;
1292 Vector<String> urlsToNotify;
1294 // Loop through the urls pending import
1295 // Remove unretained ones if database cleanup is allowed
1296 // Keep a set of ones that are retained and pending notification
1298 MutexLocker locker(m_urlAndIconLock);
1300 for (unsigned i = 0; i < urls.size(); ++i) {
1301 if (!m_retainedPageURLs.contains(urls[i])) {
1302 PageURLRecord* record = m_pageURLToRecordMap.get(urls[i]);
1303 if (record && !databaseCleanupCounter) {
1304 m_pageURLToRecordMap.remove(urls[i]);
1305 IconRecord* iconRecord = record->iconRecord();
1307 // If this page is the only remaining retainer of its icon, mark that icon for deletion and don't bother
1308 // reading anything related to it
1309 if (iconRecord && iconRecord->hasOneRef()) {
1310 m_iconURLToRecordMap.remove(iconRecord->iconURL());
1313 MutexLocker locker(m_pendingReadingLock);
1314 m_pageURLsInterestedInIcons.remove(urls[i]);
1315 m_iconsPendingReading.remove(iconRecord);
1318 MutexLocker locker(m_pendingSyncLock);
1319 m_iconsPendingSync.set(iconRecord->iconURL(), iconRecord->snapshot(true));
1326 urlsToNotify.append(urls[i]);
1331 LOG(IconDatabase, "Notifying %lu interested page URLs that their icon URL is known due to the import", static_cast<unsigned long>(urlsToNotify.size()));
1332 // Now that we don't hold any locks, perform the actual notifications
1333 for (unsigned i = 0; i < urlsToNotify.size(); ++i) {
1334 LOG(IconDatabase, "Notifying icon info known for pageURL %s", urlsToNotify[i].ascii().data());
1335 dispatchDidImportIconURLForPageURLOnMainThread(urlsToNotify[i]);
1336 if (shouldStopThreadActivity())
1342 // Notify the client that the URL import is complete in case it's managing its own pending notifications.
1343 dispatchDidFinishURLImportOnMainThread();
1345 // Notify all DocumentLoaders that were waiting for an icon load decision on the main thread
1346 callOnMainThread(notifyPendingLoadDecisionsOnMainThread, this);
1349 void* IconDatabase::syncThreadMainLoop()
1351 ASSERT_ICON_SYNC_THREAD();
1353 bool shouldReenableSuddenTermination = false;
1357 // It's possible thread termination is requested before the main loop even starts - in that case, just skip straight to cleanup
1358 while (!m_threadTerminationRequested) {
1359 m_syncLock.unlock();
1362 double timeStamp = currentTime();
1364 LOG(IconDatabase, "(THREAD) Main work loop starting");
1366 // If we should remove all icons, do it now. This is an uninteruptible procedure that we will always do before quitting if it is requested
1367 if (m_removeIconsRequested) {
1368 removeAllIconsOnThread();
1369 m_removeIconsRequested = false;
1372 // Then, if the thread should be quitting, quit now!
1373 if (m_threadTerminationRequested)
1376 bool didAnyWork = true;
1377 while (didAnyWork) {
1378 bool didWrite = writeToDatabase();
1379 if (shouldStopThreadActivity())
1382 didAnyWork = readFromDatabase();
1383 if (shouldStopThreadActivity())
1386 // Prune unretained icons after the first time we sync anything out to the database
1387 // This way, pruning won't be the only operation we perform to the database by itself
1388 // We also don't want to bother doing this if the thread should be terminating (the user is quitting)
1389 // or if private browsing is enabled
1390 // We also don't want to prune if the m_databaseCleanupCounter count is non-zero - that means someone
1391 // has asked to delay pruning
1392 static bool prunedUnretainedIcons = false;
1393 if (didWrite && !m_privateBrowsingEnabled && !prunedUnretainedIcons && !databaseCleanupCounter) {
1395 double time = currentTime();
1397 LOG(IconDatabase, "(THREAD) Starting pruneUnretainedIcons()");
1399 pruneUnretainedIcons();
1401 LOG(IconDatabase, "(THREAD) pruneUnretainedIcons() took %.4f seconds", currentTime() - time);
1403 // If pruneUnretainedIcons() returned early due to requested thread termination, its still okay
1404 // to mark prunedUnretainedIcons true because we're about to terminate anyway
1405 prunedUnretainedIcons = true;
1408 didAnyWork = didAnyWork || didWrite;
1409 if (shouldStopThreadActivity())
1414 double newstamp = currentTime();
1415 LOG(IconDatabase, "(THREAD) Main work loop ran for %.4f seconds, %s requested to terminate", newstamp - timeStamp, shouldStopThreadActivity() ? "was" : "was not");
1420 // There is some condition that is asking us to stop what we're doing now and handle a special case
1421 // This is either removing all icons, or shutting down the thread to quit the app
1422 // We handle those at the top of this main loop so continue to jump back up there
1423 if (shouldStopThreadActivity())
1426 if (shouldReenableSuddenTermination) {
1427 // The following is balanced by the call to disableSuddenTermination in the
1428 // wakeSyncThread function. Any time we wait on the condition, we also have
1429 // to enableSuddenTermation, after doing the next batch of work.
1430 ASSERT(m_disabledSuddenTerminationForSyncThread);
1431 enableSuddenTermination();
1432 m_disabledSuddenTerminationForSyncThread = false;
1435 m_syncCondition.wait(m_syncLock);
1437 shouldReenableSuddenTermination = true;
1440 m_syncLock.unlock();
1442 // Thread is terminating at this point
1443 cleanupSyncThread();
1445 if (shouldReenableSuddenTermination) {
1446 // The following is balanced by the call to disableSuddenTermination in the
1447 // wakeSyncThread function. Any time we wait on the condition, we also have
1448 // to enableSuddenTermation, after doing the next batch of work.
1449 ASSERT(m_disabledSuddenTerminationForSyncThread);
1450 enableSuddenTermination();
1451 m_disabledSuddenTerminationForSyncThread = false;
1457 bool IconDatabase::readFromDatabase()
1459 ASSERT_ICON_SYNC_THREAD();
1462 double timeStamp = currentTime();
1465 bool didAnyWork = false;
1467 // We'll make a copy of the sets of things that need to be read. Then we'll verify at the time of updating the record that it still wants to be updated
1468 // This way we won't hold the lock for a long period of time
1469 Vector<IconRecord*> icons;
1471 MutexLocker locker(m_pendingReadingLock);
1472 icons.appendRange(m_iconsPendingReading.begin(), m_iconsPendingReading.end());
1475 // Keep track of icons we actually read to notify them of the new icon
1476 HashSet<String> urlsToNotify;
1478 for (unsigned i = 0; i < icons.size(); ++i) {
1480 RefPtr<SharedBuffer> imageData = getImageDataForIconURLFromSQLDatabase(icons[i]->iconURL());
1481 imageData->setMutexForVerifier(m_urlAndIconLock);
1483 // Verify this icon still wants to be read from disk
1485 MutexLocker urlLocker(m_urlAndIconLock);
1487 MutexLocker readLocker(m_pendingReadingLock);
1489 if (m_iconsPendingReading.contains(icons[i])) {
1491 icons[i]->setImageData(imageData.release());
1493 // Remove this icon from the set that needs to be read
1494 m_iconsPendingReading.remove(icons[i]);
1496 // We have a set of all Page URLs that retain this icon as well as all PageURLs waiting for an icon
1497 // We want to find the intersection of these two sets to notify them
1498 // Check the sizes of these two sets to minimize the number of iterations
1499 const HashSet<String>* outerHash;
1500 const HashSet<String>* innerHash;
1502 if (icons[i]->retainingPageURLs().size() > m_pageURLsInterestedInIcons.size()) {
1503 outerHash = &m_pageURLsInterestedInIcons;
1504 innerHash = &(icons[i]->retainingPageURLs());
1506 innerHash = &m_pageURLsInterestedInIcons;
1507 outerHash = &(icons[i]->retainingPageURLs());
1510 HashSet<String>::const_iterator iter = outerHash->begin();
1511 HashSet<String>::const_iterator end = outerHash->end();
1512 for (; iter != end; ++iter) {
1513 if (innerHash->contains(*iter)) {
1514 LOG(IconDatabase, "%s is interesting in the icon we just read. Adding it to the list and removing it from the interested set", urlForLogging(*iter).ascii().data());
1515 urlsToNotify.add(*iter);
1518 // If we ever get to the point were we've seen every url interested in this icon, break early
1519 if (urlsToNotify.size() == m_pageURLsInterestedInIcons.size())
1523 // We don't need to notify a PageURL twice, so all the ones we're about to notify can be removed from the interested set
1524 if (urlsToNotify.size() == m_pageURLsInterestedInIcons.size())
1525 m_pageURLsInterestedInIcons.clear();
1527 iter = urlsToNotify.begin();
1528 end = urlsToNotify.end();
1529 for (; iter != end; ++iter)
1530 m_pageURLsInterestedInIcons.remove(*iter);
1536 if (shouldStopThreadActivity())
1539 // Informal testing shows that draining the autorelease pool every 25 iterations is about as low as we can go
1540 // before performance starts to drop off, but we don't want to increase this number because then accumulated memory usage will go up
1541 AutodrainedPool pool(25);
1543 // Now that we don't hold any locks, perform the actual notifications
1544 HashSet<String>::iterator iter = urlsToNotify.begin();
1545 HashSet<String>::iterator end = urlsToNotify.end();
1546 for (unsigned iteration = 0; iter != end; ++iter, ++iteration) {
1547 LOG(IconDatabase, "Notifying icon received for pageURL %s", urlForLogging(*iter).ascii().data());
1548 dispatchDidImportIconDataForPageURLOnMainThread(*iter);
1549 if (shouldStopThreadActivity())
1555 LOG(IconDatabase, "Done notifying %i pageURLs who just received their icons", urlsToNotify.size());
1556 urlsToNotify.clear();
1558 if (shouldStopThreadActivity())
1562 LOG(IconDatabase, "Reading from database took %.4f seconds", currentTime() - timeStamp);
1567 bool IconDatabase::writeToDatabase()
1569 ASSERT_ICON_SYNC_THREAD();
1572 double timeStamp = currentTime();
1575 bool didAnyWork = false;
1577 // We can copy the current work queue then clear it out - If any new work comes in while we're writing out,
1578 // we'll pick it up on the next pass. This greatly simplifies the locking strategy for this method and remains cohesive with changes
1579 // asked for by the database on the main thread
1581 MutexLocker locker(m_urlAndIconLock);
1582 Vector<IconSnapshot> iconSnapshots;
1583 Vector<PageURLSnapshot> pageSnapshots;
1585 MutexLocker locker(m_pendingSyncLock);
1587 iconSnapshots.appendRange(m_iconsPendingSync.begin().values(), m_iconsPendingSync.end().values());
1588 m_iconsPendingSync.clear();
1590 pageSnapshots.appendRange(m_pageURLsPendingSync.begin().values(), m_pageURLsPendingSync.end().values());
1591 m_pageURLsPendingSync.clear();
1594 if (iconSnapshots.size() || pageSnapshots.size())
1597 SQLiteTransaction syncTransaction(m_syncDB);
1598 syncTransaction.begin();
1600 for (unsigned i = 0; i < iconSnapshots.size(); ++i) {
1601 writeIconSnapshotToSQLDatabase(iconSnapshots[i]);
1602 LOG(IconDatabase, "Wrote IconRecord for IconURL %s with timeStamp of %i to the DB", urlForLogging(iconSnapshots[i].iconURL()).ascii().data(), iconSnapshots[i].timestamp());
1605 for (unsigned i = 0; i < pageSnapshots.size(); ++i) {
1606 // If the icon URL is empty, this page is meant to be deleted
1607 // ASSERTs are sanity checks to make sure the mappings exist if they should and don't if they shouldn't
1608 if (pageSnapshots[i].iconURL().isEmpty())
1609 removePageURLFromSQLDatabase(pageSnapshots[i].pageURL());
1611 setIconURLForPageURLInSQLDatabase(pageSnapshots[i].iconURL(), pageSnapshots[i].pageURL());
1612 LOG(IconDatabase, "Committed IconURL for PageURL %s to database", urlForLogging(pageSnapshots[i].pageURL()).ascii().data());
1615 syncTransaction.commit();
1618 // Check to make sure there are no dangling PageURLs - If there are, we want to output one log message but not spam the console potentially every few seconds
1620 checkForDanglingPageURLs(false);
1622 LOG(IconDatabase, "Updating the database took %.4f seconds", currentTime() - timeStamp);
1627 void IconDatabase::pruneUnretainedIcons()
1629 ASSERT_ICON_SYNC_THREAD();
1634 // This method should only be called once per run
1635 ASSERT(!m_initialPruningComplete);
1637 // This method relies on having read in all page URLs from the database earlier.
1638 ASSERT(m_iconURLImportComplete);
1640 // Get the known PageURLs from the db, and record the ID of any that are not in the retain count set.
1641 Vector<int64_t> pageIDsToDelete;
1643 SQLiteStatement pageSQL(m_syncDB, "SELECT rowid, url FROM PageURL;");
1647 while ((result = pageSQL.step()) == SQLResultRow) {
1648 MutexLocker locker(m_urlAndIconLock);
1649 if (!m_pageURLToRecordMap.contains(pageSQL.getColumnText(1)))
1650 pageIDsToDelete.append(pageSQL.getColumnInt64(0));
1653 if (result != SQLResultDone)
1654 LOG_ERROR("Error reading PageURL table from on-disk DB");
1657 // Delete page URLs that were in the table, but not in our retain count set.
1658 size_t numToDelete = pageIDsToDelete.size();
1660 SQLiteTransaction pruningTransaction(m_syncDB);
1661 pruningTransaction.begin();
1663 SQLiteStatement pageDeleteSQL(m_syncDB, "DELETE FROM PageURL WHERE rowid = (?);");
1664 pageDeleteSQL.prepare();
1665 for (size_t i = 0; i < numToDelete; ++i) {
1667 LOG(IconDatabase, "Pruning page with rowid %I64i from disk", static_cast<long long>(pageIDsToDelete[i]));
1669 LOG(IconDatabase, "Pruning page with rowid %lli from disk", static_cast<long long>(pageIDsToDelete[i]));
1671 pageDeleteSQL.bindInt64(1, pageIDsToDelete[i]);
1672 int result = pageDeleteSQL.step();
1673 if (result != SQLResultDone)
1675 LOG_ERROR("Unabled to delete page with id %I64i from disk", static_cast<long long>(pageIDsToDelete[i]));
1677 LOG_ERROR("Unabled to delete page with id %lli from disk", static_cast<long long>(pageIDsToDelete[i]));
1679 pageDeleteSQL.reset();
1681 // If the thread was asked to terminate, we should commit what pruning we've done so far, figuring we can
1682 // finish the rest later (hopefully)
1683 if (shouldStopThreadActivity()) {
1684 pruningTransaction.commit();
1688 pruningTransaction.commit();
1689 pageDeleteSQL.finalize();
1692 // Deleting unreferenced icons from the Icon tables has to be atomic -
1693 // If the user quits while these are taking place, they might have to wait. Thankfully this will rarely be an issue
1694 // A user on a network home directory with a wildly inconsistent database might see quite a pause...
1696 SQLiteTransaction pruningTransaction(m_syncDB);
1697 pruningTransaction.begin();
1699 // Wipe Icons that aren't retained
1700 if (!m_syncDB.executeCommand("DELETE FROM IconData WHERE iconID NOT IN (SELECT iconID FROM PageURL);"))
1701 LOG_ERROR("Failed to execute SQL to prune unretained icons from the on-disk IconData table");
1702 if (!m_syncDB.executeCommand("DELETE FROM IconInfo WHERE iconID NOT IN (SELECT iconID FROM PageURL);"))
1703 LOG_ERROR("Failed to execute SQL to prune unretained icons from the on-disk IconInfo table");
1705 pruningTransaction.commit();
1707 checkForDanglingPageURLs(true);
1709 m_initialPruningComplete = true;
1712 void IconDatabase::checkForDanglingPageURLs(bool pruneIfFound)
1714 ASSERT_ICON_SYNC_THREAD();
1716 // This check can be relatively expensive so we don't do it in a release build unless the caller has asked us to prune any dangling
1717 // entries. We also don't want to keep performing this check and reporting this error if it has already found danglers before so we
1718 // keep track of whether we've found any. We skip the check in the release build pretending to have already found danglers already.
1720 static bool danglersFound = true;
1722 static bool danglersFound = false;
1725 if ((pruneIfFound || !danglersFound) && SQLiteStatement(m_syncDB, "SELECT url FROM PageURL WHERE PageURL.iconID NOT IN (SELECT iconID FROM IconInfo) LIMIT 1;").returnsAtLeastOneResult()) {
1726 danglersFound = true;
1727 LOG(IconDatabase, "Dangling PageURL entries found");
1728 if (pruneIfFound && !m_syncDB.executeCommand("DELETE FROM PageURL WHERE iconID NOT IN (SELECT iconID FROM IconInfo);"))
1729 LOG(IconDatabase, "Unable to prune dangling PageURLs");
1733 void IconDatabase::removeAllIconsOnThread()
1735 ASSERT_ICON_SYNC_THREAD();
1737 LOG(IconDatabase, "Removing all icons on the sync thread");
1739 // Delete all the prepared statements so they can start over
1740 deleteAllPreparedStatements();
1742 // To reset the on-disk database, we'll wipe all its tables then vacuum it
1743 // This is easier and safer than closing it, deleting the file, and recreating from scratch
1744 m_syncDB.clearAllTables();
1745 m_syncDB.runVacuumCommand();
1746 createDatabaseTables(m_syncDB);
1748 LOG(IconDatabase, "Dispatching notification that we removed all icons");
1749 dispatchDidRemoveAllIconsOnMainThread();
1752 void IconDatabase::deleteAllPreparedStatements()
1754 ASSERT_ICON_SYNC_THREAD();
1756 m_setIconIDForPageURLStatement.clear();
1757 m_removePageURLStatement.clear();
1758 m_getIconIDForIconURLStatement.clear();
1759 m_getImageDataForIconURLStatement.clear();
1760 m_addIconToIconInfoStatement.clear();
1761 m_addIconToIconDataStatement.clear();
1762 m_getImageDataStatement.clear();
1763 m_deletePageURLsForIconURLStatement.clear();
1764 m_deleteIconFromIconInfoStatement.clear();
1765 m_deleteIconFromIconDataStatement.clear();
1766 m_updateIconInfoStatement.clear();
1767 m_updateIconDataStatement.clear();
1768 m_setIconInfoStatement.clear();
1769 m_setIconDataStatement.clear();
1772 void* IconDatabase::cleanupSyncThread()
1774 ASSERT_ICON_SYNC_THREAD();
1777 double timeStamp = currentTime();
1780 // If the removeIcons flag is set, remove all icons from the db.
1781 if (m_removeIconsRequested)
1782 removeAllIconsOnThread();
1784 // Sync remaining icons out
1785 LOG(IconDatabase, "(THREAD) Doing final writeout and closure of sync thread");
1788 // Close the database
1789 MutexLocker locker(m_syncLock);
1791 m_databaseDirectory = String();
1792 m_completeDatabasePath = String();
1793 deleteAllPreparedStatements();
1797 LOG(IconDatabase, "(THREAD) Final closure took %.4f seconds", currentTime() - timeStamp);
1800 m_syncThreadRunning = false;
1804 bool IconDatabase::imported()
1806 ASSERT_ICON_SYNC_THREAD();
1808 if (m_isImportedSet)
1811 SQLiteStatement query(m_syncDB, "SELECT IconDatabaseInfo.value FROM IconDatabaseInfo WHERE IconDatabaseInfo.key = \"ImportedSafari2Icons\";");
1812 if (query.prepare() != SQLResultOk) {
1813 LOG_ERROR("Unable to prepare imported statement");
1817 int result = query.step();
1818 if (result == SQLResultRow)
1819 result = query.getColumnInt(0);
1821 if (result != SQLResultDone)
1822 LOG_ERROR("imported statement failed");
1826 m_isImportedSet = true;
1827 return m_imported = result;
1830 void IconDatabase::setImported(bool import)
1832 ASSERT_ICON_SYNC_THREAD();
1834 m_imported = import;
1835 m_isImportedSet = true;
1837 String queryString = import ?
1838 "INSERT INTO IconDatabaseInfo (key, value) VALUES (\"ImportedSafari2Icons\", 1);" :
1839 "INSERT INTO IconDatabaseInfo (key, value) VALUES (\"ImportedSafari2Icons\", 0);";
1841 SQLiteStatement query(m_syncDB, queryString);
1843 if (query.prepare() != SQLResultOk) {
1844 LOG_ERROR("Unable to prepare set imported statement");
1848 if (query.step() != SQLResultDone)
1849 LOG_ERROR("set imported statement failed");
1852 // readySQLiteStatement() handles two things
1853 // 1 - If the SQLDatabase& argument is different, the statement must be destroyed and remade. This happens when the user
1854 // switches to and from private browsing
1855 // 2 - Lazy construction of the Statement in the first place, in case we've never made this query before
1856 inline void readySQLiteStatement(OwnPtr<SQLiteStatement>& statement, SQLiteDatabase& db, const String& str)
1858 if (statement && (statement->database() != &db || statement->isExpired())) {
1859 if (statement->isExpired())
1860 LOG(IconDatabase, "SQLiteStatement associated with %s is expired", str.ascii().data());
1864 statement = adoptPtr(new SQLiteStatement(db, str));
1865 if (statement->prepare() != SQLResultOk)
1866 LOG_ERROR("Preparing statement %s failed", str.ascii().data());
1870 void IconDatabase::setIconURLForPageURLInSQLDatabase(const String& iconURL, const String& pageURL)
1872 ASSERT_ICON_SYNC_THREAD();
1874 int64_t iconID = getIconIDForIconURLFromSQLDatabase(iconURL);
1877 iconID = addIconURLToSQLDatabase(iconURL);
1880 LOG_ERROR("Failed to establish an ID for iconURL %s", urlForLogging(iconURL).ascii().data());
1885 setIconIDForPageURLInSQLDatabase(iconID, pageURL);
1888 void IconDatabase::setIconIDForPageURLInSQLDatabase(int64_t iconID, const String& pageURL)
1890 ASSERT_ICON_SYNC_THREAD();
1892 readySQLiteStatement(m_setIconIDForPageURLStatement, m_syncDB, "INSERT INTO PageURL (url, iconID) VALUES ((?), ?);");
1893 m_setIconIDForPageURLStatement->bindText(1, pageURL);
1894 m_setIconIDForPageURLStatement->bindInt64(2, iconID);
1896 int result = m_setIconIDForPageURLStatement->step();
1897 if (result != SQLResultDone) {
1899 LOG_ERROR("setIconIDForPageURLQuery failed for url %s", urlForLogging(pageURL).ascii().data());
1902 m_setIconIDForPageURLStatement->reset();
1905 void IconDatabase::removePageURLFromSQLDatabase(const String& pageURL)
1907 ASSERT_ICON_SYNC_THREAD();
1909 readySQLiteStatement(m_removePageURLStatement, m_syncDB, "DELETE FROM PageURL WHERE url = (?);");
1910 m_removePageURLStatement->bindText(1, pageURL);
1912 if (m_removePageURLStatement->step() != SQLResultDone)
1913 LOG_ERROR("removePageURLFromSQLDatabase failed for url %s", urlForLogging(pageURL).ascii().data());
1915 m_removePageURLStatement->reset();
1919 int64_t IconDatabase::getIconIDForIconURLFromSQLDatabase(const String& iconURL)
1921 ASSERT_ICON_SYNC_THREAD();
1923 readySQLiteStatement(m_getIconIDForIconURLStatement, m_syncDB, "SELECT IconInfo.iconID FROM IconInfo WHERE IconInfo.url = (?);");
1924 m_getIconIDForIconURLStatement->bindText(1, iconURL);
1926 int64_t result = m_getIconIDForIconURLStatement->step();
1927 if (result == SQLResultRow)
1928 result = m_getIconIDForIconURLStatement->getColumnInt64(0);
1930 if (result != SQLResultDone)
1931 LOG_ERROR("getIconIDForIconURLFromSQLDatabase failed for url %s", urlForLogging(iconURL).ascii().data());
1935 m_getIconIDForIconURLStatement->reset();
1939 int64_t IconDatabase::addIconURLToSQLDatabase(const String& iconURL)
1941 ASSERT_ICON_SYNC_THREAD();
1943 // There would be a transaction here to make sure these two inserts are atomic
1944 // In practice the only caller of this method is always wrapped in a transaction itself so placing another
1945 // here is unnecessary
1947 readySQLiteStatement(m_addIconToIconInfoStatement, m_syncDB, "INSERT INTO IconInfo (url, stamp) VALUES (?, 0);");
1948 m_addIconToIconInfoStatement->bindText(1, iconURL);
1950 int result = m_addIconToIconInfoStatement->step();
1951 m_addIconToIconInfoStatement->reset();
1952 if (result != SQLResultDone) {
1953 LOG_ERROR("addIconURLToSQLDatabase failed to insert %s into IconInfo", urlForLogging(iconURL).ascii().data());
1956 int64_t iconID = m_syncDB.lastInsertRowID();
1958 readySQLiteStatement(m_addIconToIconDataStatement, m_syncDB, "INSERT INTO IconData (iconID, data) VALUES (?, ?);");
1959 m_addIconToIconDataStatement->bindInt64(1, iconID);
1961 result = m_addIconToIconDataStatement->step();
1962 m_addIconToIconDataStatement->reset();
1963 if (result != SQLResultDone) {
1964 LOG_ERROR("addIconURLToSQLDatabase failed to insert %s into IconData", urlForLogging(iconURL).ascii().data());
1971 PassRefPtr<SharedBuffer> IconDatabase::getImageDataForIconURLFromSQLDatabase(const String& iconURL)
1973 ASSERT_ICON_SYNC_THREAD();
1975 RefPtr<SharedBuffer> imageData;
1977 readySQLiteStatement(m_getImageDataForIconURLStatement, m_syncDB, "SELECT IconData.data FROM IconData WHERE IconData.iconID IN (SELECT iconID FROM IconInfo WHERE IconInfo.url = (?));");
1978 m_getImageDataForIconURLStatement->bindText(1, iconURL);
1980 int result = m_getImageDataForIconURLStatement->step();
1981 if (result == SQLResultRow) {
1983 m_getImageDataForIconURLStatement->getColumnBlobAsVector(0, data);
1984 imageData = SharedBuffer::create(data.data(), data.size());
1985 } else if (result != SQLResultDone)
1986 LOG_ERROR("getImageDataForIconURLFromSQLDatabase failed for url %s", urlForLogging(iconURL).ascii().data());
1988 m_getImageDataForIconURLStatement->reset();
1990 return imageData.release();
1993 void IconDatabase::removeIconFromSQLDatabase(const String& iconURL)
1995 ASSERT_ICON_SYNC_THREAD();
1997 if (iconURL.isEmpty())
2000 // There would be a transaction here to make sure these removals are atomic
2001 // In practice the only caller of this method is always wrapped in a transaction itself so placing another here is unnecessary
2003 // It's possible this icon is not in the database because of certain rapid browsing patterns (such as a stress test) where the
2004 // icon is marked to be added then marked for removal before it is ever written to disk. No big deal, early return
2005 int64_t iconID = getIconIDForIconURLFromSQLDatabase(iconURL);
2009 readySQLiteStatement(m_deletePageURLsForIconURLStatement, m_syncDB, "DELETE FROM PageURL WHERE PageURL.iconID = (?);");
2010 m_deletePageURLsForIconURLStatement->bindInt64(1, iconID);
2012 if (m_deletePageURLsForIconURLStatement->step() != SQLResultDone)
2013 LOG_ERROR("m_deletePageURLsForIconURLStatement failed for url %s", urlForLogging(iconURL).ascii().data());
2015 readySQLiteStatement(m_deleteIconFromIconInfoStatement, m_syncDB, "DELETE FROM IconInfo WHERE IconInfo.iconID = (?);");
2016 m_deleteIconFromIconInfoStatement->bindInt64(1, iconID);
2018 if (m_deleteIconFromIconInfoStatement->step() != SQLResultDone)
2019 LOG_ERROR("m_deleteIconFromIconInfoStatement failed for url %s", urlForLogging(iconURL).ascii().data());
2021 readySQLiteStatement(m_deleteIconFromIconDataStatement, m_syncDB, "DELETE FROM IconData WHERE IconData.iconID = (?);");
2022 m_deleteIconFromIconDataStatement->bindInt64(1, iconID);
2024 if (m_deleteIconFromIconDataStatement->step() != SQLResultDone)
2025 LOG_ERROR("m_deleteIconFromIconDataStatement failed for url %s", urlForLogging(iconURL).ascii().data());
2027 m_deletePageURLsForIconURLStatement->reset();
2028 m_deleteIconFromIconInfoStatement->reset();
2029 m_deleteIconFromIconDataStatement->reset();
2032 void IconDatabase::writeIconSnapshotToSQLDatabase(const IconSnapshot& snapshot)
2034 ASSERT_ICON_SYNC_THREAD();
2036 if (snapshot.iconURL().isEmpty())
2039 // A nulled out timestamp and data means this icon is destined to be deleted - do that instead of writing it out
2040 if (!snapshot.timestamp() && !snapshot.data()) {
2041 LOG(IconDatabase, "Removing %s from on-disk database", urlForLogging(snapshot.iconURL()).ascii().data());
2042 removeIconFromSQLDatabase(snapshot.iconURL());
2046 // There would be a transaction here to make sure these removals are atomic
2047 // In practice the only caller of this method is always wrapped in a transaction itself so placing another here is unnecessary
2049 // Get the iconID for this url
2050 int64_t iconID = getIconIDForIconURLFromSQLDatabase(snapshot.iconURL());
2052 // If there is already an iconID in place, update the database.
2053 // Otherwise, insert new records
2055 readySQLiteStatement(m_updateIconInfoStatement, m_syncDB, "UPDATE IconInfo SET stamp = ?, url = ? WHERE iconID = ?;");
2056 m_updateIconInfoStatement->bindInt64(1, snapshot.timestamp());
2057 m_updateIconInfoStatement->bindText(2, snapshot.iconURL());
2058 m_updateIconInfoStatement->bindInt64(3, iconID);
2060 if (m_updateIconInfoStatement->step() != SQLResultDone)
2061 LOG_ERROR("Failed to update icon info for url %s", urlForLogging(snapshot.iconURL()).ascii().data());
2063 m_updateIconInfoStatement->reset();
2065 readySQLiteStatement(m_updateIconDataStatement, m_syncDB, "UPDATE IconData SET data = ? WHERE iconID = ?;");
2066 m_updateIconDataStatement->bindInt64(2, iconID);
2068 // If we *have* image data, bind it to this statement - Otherwise bind "null" for the blob data,
2069 // signifying that this icon doesn't have any data
2070 if (snapshot.data() && snapshot.data()->size())
2071 m_updateIconDataStatement->bindBlob(1, snapshot.data()->data(), snapshot.data()->size());
2073 m_updateIconDataStatement->bindNull(1);
2075 if (m_updateIconDataStatement->step() != SQLResultDone)
2076 LOG_ERROR("Failed to update icon data for url %s", urlForLogging(snapshot.iconURL()).ascii().data());
2078 m_updateIconDataStatement->reset();
2080 readySQLiteStatement(m_setIconInfoStatement, m_syncDB, "INSERT INTO IconInfo (url,stamp) VALUES (?, ?);");
2081 m_setIconInfoStatement->bindText(1, snapshot.iconURL());
2082 m_setIconInfoStatement->bindInt64(2, snapshot.timestamp());
2084 if (m_setIconInfoStatement->step() != SQLResultDone)
2085 LOG_ERROR("Failed to set icon info for url %s", urlForLogging(snapshot.iconURL()).ascii().data());
2087 m_setIconInfoStatement->reset();
2089 int64_t iconID = m_syncDB.lastInsertRowID();
2091 readySQLiteStatement(m_setIconDataStatement, m_syncDB, "INSERT INTO IconData (iconID, data) VALUES (?, ?);");
2092 m_setIconDataStatement->bindInt64(1, iconID);
2094 // If we *have* image data, bind it to this statement - Otherwise bind "null" for the blob data,
2095 // signifying that this icon doesn't have any data
2096 if (snapshot.data() && snapshot.data()->size())
2097 m_setIconDataStatement->bindBlob(2, snapshot.data()->data(), snapshot.data()->size());
2099 m_setIconDataStatement->bindNull(2);
2101 if (m_setIconDataStatement->step() != SQLResultDone)
2102 LOG_ERROR("Failed to set icon data for url %s", urlForLogging(snapshot.iconURL()).ascii().data());
2104 m_setIconDataStatement->reset();
2108 bool IconDatabase::wasExcludedFromBackup()
2110 ASSERT_ICON_SYNC_THREAD();
2112 return SQLiteStatement(m_syncDB, "SELECT value FROM IconDatabaseInfo WHERE key = 'ExcludedFromBackup';").getColumnInt(0);
2115 void IconDatabase::setWasExcludedFromBackup()
2117 ASSERT_ICON_SYNC_THREAD();
2119 SQLiteStatement(m_syncDB, "INSERT INTO IconDatabaseInfo (key, value) VALUES ('ExcludedFromBackup', 1)").executeCommand();
2122 class ClientWorkItem {
2124 ClientWorkItem(IconDatabaseClient* client)
2127 virtual void performWork() = 0;
2128 virtual ~ClientWorkItem() { }
2131 IconDatabaseClient* m_client;
2134 class ImportedIconURLForPageURLWorkItem : public ClientWorkItem {
2136 ImportedIconURLForPageURLWorkItem(IconDatabaseClient* client, const String& pageURL)
2137 : ClientWorkItem(client)
2138 , m_pageURL(new String(pageURL.threadsafeCopy()))
2141 virtual ~ImportedIconURLForPageURLWorkItem()
2146 virtual void performWork()
2149 m_client->didImportIconURLForPageURL(*m_pageURL);
2157 class ImportedIconDataForPageURLWorkItem : public ClientWorkItem {
2159 ImportedIconDataForPageURLWorkItem(IconDatabaseClient* client, const String& pageURL)
2160 : ClientWorkItem(client)
2161 , m_pageURL(new String(pageURL.threadsafeCopy()))
2164 virtual ~ImportedIconDataForPageURLWorkItem()
2169 virtual void performWork()
2172 m_client->didImportIconDataForPageURL(*m_pageURL);
2180 class RemovedAllIconsWorkItem : public ClientWorkItem {
2182 RemovedAllIconsWorkItem(IconDatabaseClient* client)
2183 : ClientWorkItem(client)
2186 virtual void performWork()
2189 m_client->didRemoveAllIcons();
2194 class FinishedURLImport : public ClientWorkItem {
2196 FinishedURLImport(IconDatabaseClient* client)
2197 : ClientWorkItem(client)
2200 virtual void performWork()
2203 m_client->didFinishURLImport();
2208 static void performWorkItem(void* context)
2210 ClientWorkItem* item = static_cast<ClientWorkItem*>(context);
2211 item->performWork();
2215 void IconDatabase::dispatchDidImportIconURLForPageURLOnMainThread(const String& pageURL)
2217 ASSERT_ICON_SYNC_THREAD();
2219 ImportedIconURLForPageURLWorkItem* work = new ImportedIconURLForPageURLWorkItem(m_client, pageURL);
2220 callOnMainThread(performWorkItem, work);
2223 void IconDatabase::dispatchDidImportIconDataForPageURLOnMainThread(const String& pageURL)
2225 ASSERT_ICON_SYNC_THREAD();
2227 ImportedIconDataForPageURLWorkItem* work = new ImportedIconDataForPageURLWorkItem(m_client, pageURL);
2228 callOnMainThread(performWorkItem, work);
2231 void IconDatabase::dispatchDidRemoveAllIconsOnMainThread()
2233 ASSERT_ICON_SYNC_THREAD();
2235 RemovedAllIconsWorkItem* work = new RemovedAllIconsWorkItem(m_client);
2236 callOnMainThread(performWorkItem, work);
2239 void IconDatabase::dispatchDidFinishURLImportOnMainThread()
2241 ASSERT_ICON_SYNC_THREAD();
2243 FinishedURLImport* work = new FinishedURLImport(m_client);
2244 callOnMainThread(performWorkItem, work);
2248 } // namespace WebCore
2250 #endif // ENABLE(ICONDATABASE)