2 * Copyright (C) 2013 Team XBMC
5 * This Program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2, or (at your option)
10 * This Program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with XBMC; see the file COPYING. If not, see
17 * <http://www.gnu.org/licenses/>.
21 #if (defined HAVE_CONFIG_H) && (!defined TARGET_WINDOWS)
25 // python.h should always be included first before any other includes
30 #include "PythonInvoker.h"
31 #include "Application.h"
32 #include "ApplicationMessenger.h"
33 #include "addons/AddonManager.h"
34 #include "dialogs/GUIDialogKaiToast.h"
35 #include "filesystem/File.h"
36 #include "filesystem/SpecialProtocol.h"
37 #include "guilib/GraphicContext.h"
38 #include "guilib/GUIWindowManager.h"
39 #include "interfaces/legacy/Addon.h"
40 #include "interfaces/python/LanguageHook.h"
41 #include "interfaces/python/PyContext.h"
42 #include "interfaces/python/pythreadstate.h"
43 #include "interfaces/python/swig.h"
44 #include "interfaces/python/XBPython.h"
45 #include "threads/SingleLock.h"
46 #if defined(TARGET_WINDOWS)
47 #include "utils/CharsetConverter.h"
48 #endif // defined(TARGET_WINDOWS)
49 #include "utils/log.h"
50 #include "utils/StringUtils.h"
51 #include "utils/URIUtils.h"
54 extern "C" FILE *fopen_utf8(const char *_Filename, const char *_Mode);
56 #define fopen_utf8 fopen
63 #define PY_PATH_SEP DELIM
65 // Time before ill-behaved scripts are terminated
66 #define PYTHON_SCRIPT_TIMEOUT 5000 // ms
69 using namespace XFILE;
73 int xbp_chdir(const char *dirname);
74 char* dll_getenv(const char* szKey);
77 #define PythonModulesSize sizeof(PythonModules) / sizeof(PythonModule)
79 CCriticalSection CPythonInvoker::s_critical;
81 static const CStdString getListOfAddonClassesAsString(XBMCAddon::AddonClass::Ref<XBMCAddon::Python::PythonLanguageHook>& languageHook)
84 CSingleLock l(*(languageHook.get()));
85 std::set<XBMCAddon::AddonClass*>& acs = languageHook->GetRegisteredAddonClasses();
86 bool firstTime = true;
87 for (std::set<XBMCAddon::AddonClass*>::iterator iter = acs.begin(); iter != acs.end(); ++iter)
93 message += (*iter)->GetClassname();
99 CPythonInvoker::CPythonInvoker(ILanguageInvocationHandler *invocationHandler)
100 : ILanguageInvoker(invocationHandler),
101 m_argc(0), m_argv(NULL),
102 m_threadState(NULL), m_stop(false)
105 CPythonInvoker::~CPythonInvoker()
107 // nothing to do for the default invoker used for registration with the
108 // CScriptInvocationManager
112 if (GetState() < InvokerStateDone)
113 CLog::Log(LOGDEBUG, "CPythonInvoker(%d): waiting for python thread \"%s\" to stop",
114 GetId(), (!m_sourceFile.empty() ? m_sourceFile.c_str() : "unknown script"));
116 g_pythonParser.PulseGlobalEvent();
120 for (unsigned int i = 0; i < m_argc; i++)
124 g_pythonParser.FinalizeScript();
127 bool CPythonInvoker::Execute(const std::string &script, const std::vector<std::string> &arguments /* = std::vector<std::string>() */)
132 if (!CFile::Exists(script))
134 CLog::Log(LOGERROR, "CPythonInvoker(%d): python script \"%s\" does not exist", GetId(), CSpecialProtocol::TranslatePath(script).c_str());
138 if (!g_pythonParser.InitializeEngine())
141 return ILanguageInvoker::Execute(script, arguments);
144 bool CPythonInvoker::execute(const std::string &script, const std::vector<std::string> &arguments)
146 // copy the code/script into a local string buffer
147 m_sourceFile = script;
149 // copy the arguments into a local buffer
150 m_argc = arguments.size();
151 m_argv = new char*[m_argc];
152 for (unsigned int i = 0; i < m_argc; i++)
154 m_argv[i] = new char[arguments.at(i).length() + 1];
155 strcpy(m_argv[i], arguments.at(i).c_str());
158 CLog::Log(LOGDEBUG, "CPythonInvoker(%d, %s): start processing", GetId(), m_sourceFile.c_str());
159 int m_Py_file_input = Py_file_input;
161 // get the global lock
162 PyEval_AcquireLock();
163 PyThreadState* state = Py_NewInterpreter();
166 PyEval_ReleaseLock();
167 CLog::Log(LOGERROR, "CPythonInvoker(%d, %s): FAILED to get thread state!", GetId(), m_sourceFile.c_str());
170 // swap in my thread state
171 PyThreadState_Swap(state);
173 XBMCAddon::AddonClass::Ref<XBMCAddon::Python::PythonLanguageHook> languageHook(new XBMCAddon::Python::PythonLanguageHook(state->interp));
174 languageHook->RegisterMe();
177 setState(InvokerStateInitialized);
179 std::string realFilename(CSpecialProtocol::TranslatePath(m_sourceFile));
180 if (realFilename == m_sourceFile)
181 CLog::Log(LOGDEBUG, "CPythonInvoker(%d, %s): the source file to load is \"%s\"", GetId(), m_sourceFile.c_str(), m_sourceFile.c_str());
183 CLog::Log(LOGDEBUG, "CPythonInvoker(%d, %s): the source file to load is \"%s\" (\"%s\")", GetId(), m_sourceFile.c_str(), m_sourceFile.c_str(), realFilename.c_str());
185 // get path from script file name and add python path's
186 // this is used for python so it will search modules from script path first
187 CStdString scriptDir = URIUtils::GetDirectory(realFilename);
188 URIUtils::RemoveSlashAtEnd(scriptDir);
191 // add on any addon modules the user has installed
192 ADDON::VECADDONS addons;
193 ADDON::CAddonMgr::Get().GetAddons(ADDON::ADDON_SCRIPT_MODULE, addons);
194 for (unsigned int i = 0; i < addons.size(); ++i)
195 addPath(CSpecialProtocol::TranslatePath(addons[i]->LibPath()));
197 // we want to use sys.path so it includes site-packages
198 // if this fails, default to using Py_GetPath
199 PyObject *sysMod(PyImport_ImportModule((char*)"sys")); // must call Py_DECREF when finished
200 PyObject *sysModDict(PyModule_GetDict(sysMod)); // borrowed ref, no need to delete
201 PyObject *pathObj(PyDict_GetItemString(sysModDict, "path")); // borrowed ref, no need to delete
203 if (pathObj != NULL && PyList_Check(pathObj))
205 for (int i = 0; i < PyList_Size(pathObj); i++)
207 PyObject *e = PyList_GetItem(pathObj, i); // borrowed ref, no need to delete
208 if (e != NULL && PyString_Check(e))
209 addNativePath(PyString_AsString(e)); // returns internal data, don't delete or modify
213 addNativePath(Py_GetPath());
215 Py_DECREF(sysMod); // release ref to sysMod
217 // set current directory and python's path.
219 PySys_SetArgv(m_argc, m_argv);
221 #ifdef TARGET_WINDOWS
222 std::string pyPathUtf8;
223 g_charsetConverter.systemToUtf8(m_pythonPath, pyPathUtf8, false);
224 CLog::Log(LOGDEBUG, "CPythonInvoker(%d, %s): setting the Python path to %s", GetId(), m_sourceFile.c_str(), pyPathUtf8.c_str());
225 #else // ! TARGET_WINDOWS
226 CLog::Log(LOGDEBUG, "CPythonInvoker(%d, %s): setting the Python path to %s", GetId(), m_sourceFile.c_str(), m_pythonPath.c_str());
227 #endif // ! TARGET_WINDOWS
228 PySys_SetPath((char *)m_pythonPath.c_str());
230 CLog::Log(LOGDEBUG, "CPythonInvoker(%d, %s): entering source directory %s", GetId(), m_sourceFile.c_str(), scriptDir.c_str());
231 PyObject* module = PyImport_AddModule((char*)"__main__");
232 PyObject* moduleDict = PyModule_GetDict(module);
234 // when we are done initing we store thread state so we can be aborted
235 PyThreadState_Swap(NULL);
236 PyEval_ReleaseLock();
238 // we need to check if we was asked to abort before we had inited
239 bool stopping = false;
240 { CSingleLock lock(m_critical);
241 m_threadState = state;
245 PyEval_AcquireLock();
246 PyThreadState_Swap(state);
253 // run script from file
254 // We need to have python open the file because on Windows the DLL that python
255 // is linked against may not be the DLL that xbmc is linked against so
256 // passing a FILE* to python from an fopen has the potential to crash.
257 std::string nativeFilename(realFilename); // filename in system encoding
258 #ifdef TARGET_WINDOWS
259 if (!g_charsetConverter.utf8ToSystem(nativeFilename, true))
261 CLog::Log(LOGERROR, "CPythonInvoker(%d, %s): can't convert filename \"%s\" to system encoding", GetId(), m_sourceFile.c_str(), realFilename.c_str());
265 PyObject* file = PyFile_FromString((char *)nativeFilename.c_str(), (char*)"r");
266 FILE *fp = PyFile_AsFile(file);
270 PyObject *f = PyString_FromString(nativeFilename.c_str());
271 PyDict_SetItemString(moduleDict, "__file__", f);
273 onPythonModuleInitialization(moduleDict);
276 setState(InvokerStateRunning);
277 XBMCAddon::Python::PyContext pycontext; // this is a guard class that marks this callstack as being in a python context
278 PyRun_FileExFlags(fp, nativeFilename.c_str(), m_Py_file_input, moduleDict, moduleDict, 1, NULL);
281 CLog::Log(LOGERROR, "CPythonInvoker(%d, %s): %s not found!", GetId(), m_sourceFile.c_str(), m_sourceFile.c_str());
283 catch (const XbmcCommons::Exception& e)
285 setState(InvokerStateFailed);
291 setState(InvokerStateFailed);
292 CLog::Log(LOGERROR, "CPythonInvoker(%d, %s): failure in script", GetId(), m_sourceFile.c_str());
297 bool systemExitThrown = false;
298 InvokerState stateToSet;
299 if (!failed && !PyErr_Occurred())
301 CLog::Log(LOGINFO, "CPythonInvoker(%d, %s): script successfully run", GetId(), m_sourceFile.c_str());
302 stateToSet = InvokerStateDone;
305 else if (PyErr_ExceptionMatches(PyExc_SystemExit))
307 systemExitThrown = true;
308 CLog::Log(LOGINFO, "CPythonInvoker(%d, %s): script aborted", GetId(), m_sourceFile.c_str());
309 stateToSet = InvokerStateFailed;
314 stateToSet = InvokerStateFailed;
316 // if it failed with an exception we already logged the details
319 PythonBindings::PythonToCppException e;
326 // no need to do anything else because the script has already stopped
329 setState(stateToSet);
333 PyObject *m = PyImport_AddModule((char*)"xbmc");
334 if (m == NULL || PyObject_SetAttrString(m, (char*)"abortRequested", PyBool_FromLong(1)))
335 CLog::Log(LOGERROR, "CPythonInvoker(%d, %s): failed to set abortRequested", GetId(), m_sourceFile.c_str());
337 // make sure all sub threads have finished
338 for (PyThreadState* s = state->interp->tstate_head, *old = NULL; s;)
347 CLog::Log(LOGINFO, "CPythonInvoker(%d, %s): waiting on thread %"PRIu64, GetId(), m_sourceFile.c_str(), (uint64_t)s->thread_id);
351 CPyThreadState pyState;
355 s = state->interp->tstate_head;
358 // pending calls must be cleared out
359 XBMCAddon::RetardedAsynchCallbackHandler::clearPendingCalls(state);
361 PyThreadState_Swap(NULL);
362 PyEval_ReleaseLock();
364 // set stopped event - this allows ::stop to run and kill remaining threads
365 // this event has to be fired without holding m_critical
366 // also the GIL (PyEval_AcquireLock) must not be held
367 // if not obeyed there is still no deadlock because ::stop waits with timeout (smart one!)
368 m_stoppedEvent.Set();
370 { CSingleLock lock(m_critical);
371 m_threadState = NULL;
374 PyEval_AcquireLock();
375 PyThreadState_Swap(state);
377 onDeinitialization();
379 // run the gc before finishing
381 // if the script exited by throwing a SystemExit excepton then going back
382 // into the interpreter causes this python bug to get hit:
383 // http://bugs.python.org/issue10582
384 // and that causes major failures. So we are not going to go back in
385 // to run the GC if that's the case.
386 if (!m_stop && languageHook->HasRegisteredAddonClasses() && !systemExitThrown &&
387 PyRun_SimpleString(GC_SCRIPT) == -1)
388 CLog::Log(LOGERROR, "CPythonInvoker(%d, %s): failed to run the gc to clean up after running prior to shutting down the Interpreter", GetId(), m_sourceFile.c_str());
390 Py_EndInterpreter(state);
392 // If we still have objects left around, produce an error message detailing what's been left behind
393 if (languageHook->HasRegisteredAddonClasses())
394 CLog::Log(LOGWARNING, "CPythonInvoker(%d, %s): the python script \"%s\" has left several "
395 "classes in memory that we couldn't clean up. The classes include: %s",
396 GetId(), m_sourceFile.c_str(), m_sourceFile.c_str(), getListOfAddonClassesAsString(languageHook).c_str());
398 // unregister the language hook
399 languageHook->UnregisterMe();
401 PyEval_ReleaseLock();
403 setState(stateToSet);
408 bool CPythonInvoker::stop(bool abort)
410 CSingleLock lock(m_critical);
416 setState(InvokerStateStopping);
418 if (m_threadState != NULL)
420 PyEval_AcquireLock();
421 PyThreadState* old = PyThreadState_Swap((PyThreadState*)m_threadState);
423 //tell xbmc.Monitor to call onAbortRequested()
425 g_pythonParser.OnAbortRequested(m_addon->ID());
428 m = PyImport_AddModule((char*)"xbmc");
429 if (m == NULL || PyObject_SetAttrString(m, (char*)"abortRequested", PyBool_FromLong(1)))
430 CLog::Log(LOGERROR, "CPythonInvoker(%d, %s): failed to set abortRequested", GetId(), m_sourceFile.c_str());
432 PyThreadState_Swap(old);
434 PyEval_ReleaseLock();
436 XbmcThreads::EndTime timeout(PYTHON_SCRIPT_TIMEOUT);
437 while (!m_stoppedEvent.WaitMSec(15))
439 if (timeout.IsTimePast())
441 CLog::Log(LOGERROR, "CPythonInvoker(%d, %s): script didn't stop in %d seconds - let's kill it", GetId(), m_sourceFile.c_str(), PYTHON_SCRIPT_TIMEOUT / 1000);
445 // We can't empty-spin in the main thread and expect scripts to be able to
446 // dismantle themselves. Python dialogs aren't normal XBMC dialogs, they rely
447 // on TMSG_GUI_PYTHON_DIALOG messages, so pump the message loop.
448 if (g_application.IsCurrentThread())
450 CSingleExit ex(g_graphicsContext);
451 CApplicationMessenger::Get().ProcessMessages();
455 // Useful for add-on performance metrics
456 if (!timeout.IsTimePast())
457 CLog::Log(LOGDEBUG, "CPythonInvoker(%d, %s): script termination took %dms", GetId(), m_sourceFile.c_str(), PYTHON_SCRIPT_TIMEOUT - timeout.MillisLeft());
459 // everything which didn't exit by now gets killed
461 // grabbing the PyLock while holding the m_critical is asking for a deadlock
462 CSingleExit ex2(m_critical);
463 PyEval_AcquireLock();
466 // Since we released the m_critical it's possible that the state is cleaned up
467 // so we need to recheck for m_threadState == NULL
468 if (m_threadState != NULL)
470 old = PyThreadState_Swap((PyThreadState*)m_threadState);
471 for (PyThreadState* state = ((PyThreadState*)m_threadState)->interp->tstate_head; state; state = state->next)
473 // Raise a SystemExit exception in python threads
474 Py_XDECREF(state->async_exc);
475 state->async_exc = PyExc_SystemExit;
476 Py_XINCREF(state->async_exc);
479 // If a dialog entered its doModal(), we need to wake it to see the exception
480 g_pythonParser.PulseGlobalEvent();
484 PyThreadState_Swap(old);
487 PyEval_ReleaseLock();
493 void CPythonInvoker::onExecutionFailed()
495 PyThreadState_Swap(NULL);
496 PyEval_ReleaseLock();
498 setState(InvokerStateFailed);
499 CLog::Log(LOGERROR, "CPythonInvoker(%d, %s): abnormally terminating python thread", GetId(), m_sourceFile.c_str());
501 CSingleLock lock(m_critical);
502 m_threadState = NULL;
504 ILanguageInvoker::onExecutionFailed();
507 std::map<std::string, CPythonInvoker::PythonModuleInitialization> CPythonInvoker::getModules() const
509 static std::map<std::string, PythonModuleInitialization> modules;
513 void CPythonInvoker::onInitialization()
517 GilSafeSingleLock lock(s_critical);
518 initializeModules(getModules());
521 // get a possible initialization script
522 const char* runscript = getInitializationScript();
523 if (runscript!= NULL && strlen(runscript) > 0)
525 // redirecting default output to debug console
526 if (PyRun_SimpleString(runscript) == -1)
527 CLog::Log(LOGFATAL, "CPythonInvoker(%d, %s): initialize error", GetId(), m_sourceFile.c_str());
531 void CPythonInvoker::onPythonModuleInitialization(void* moduleDict)
533 if (m_addon.get() == NULL || moduleDict == NULL)
536 PyObject *moduleDictionary = (PyObject *)moduleDict;
538 PyObject *pyaddonid = PyString_FromString(m_addon->ID().c_str());
539 PyDict_SetItemString(moduleDictionary, "__xbmcaddonid__", pyaddonid);
541 CStdString version = ADDON::GetXbmcApiVersionDependency(m_addon);
542 PyObject *pyxbmcapiversion = PyString_FromString(version.c_str());
543 PyDict_SetItemString(moduleDictionary, "__xbmcapiversion__", pyxbmcapiversion);
545 CLog::Log(LOGDEBUG, "CPythonInvoker(%d, %s): instantiating addon using automatically obtained id of \"%s\" dependent on version %s of the xbmc.python api",
546 GetId(), m_sourceFile.c_str(), m_addon->ID().c_str(), version.c_str());
549 void CPythonInvoker::onDeinitialization()
554 void CPythonInvoker::onError()
556 CPyThreadState releaseGil;
557 CSingleLock gc(g_graphicsContext);
559 CGUIDialogKaiToast *pDlgToast = (CGUIDialogKaiToast*)g_windowManager.GetWindow(WINDOW_DIALOG_KAI_TOAST);
560 if (pDlgToast != NULL)
564 if (m_addon.get() != NULL)
565 script = m_addon->Name();
569 URIUtils::Split(m_sourceFile.c_str(), path, script);
570 if (script.Equals("default.py"))
573 URIUtils::RemoveSlashAtEnd(path);
574 URIUtils::Split(path, path2, script);
578 desc = StringUtils::Format(g_localizeStrings.Get(2100), script.c_str());
579 pDlgToast->QueueNotification(CGUIDialogKaiToast::Error, g_localizeStrings.Get(257), desc);
583 const char* CPythonInvoker::getInitializationScript() const
588 void CPythonInvoker::initializeModules(const std::map<std::string, PythonModuleInitialization> &modules)
590 for (std::map<std::string, PythonModuleInitialization>::const_iterator module = modules.begin(); module != modules.end(); ++module)
592 if (!initializeModule(module->second))
593 CLog::Log(LOGWARNING, "CPythonInvoker(%d, %s): unable to initialize python module \"%s\"", GetId(), m_sourceFile.c_str(), module->first.c_str());
597 bool CPythonInvoker::initializeModule(PythonModuleInitialization module)
606 void CPythonInvoker::addPath(const std::string& path)
608 #if defined(TARGET_WINDOWS)
612 std::string nativePath(path);
613 if (!g_charsetConverter.utf8ToSystem(nativePath, true))
615 CLog::Log(LOGERROR, "%s: can't convert UTF-8 path \"%s\" to system encoding", __FUNCTION__, path.c_str());
618 addNativePath(nativePath);
621 #endif // defined(TARGET_WINDOWS)
624 void CPythonInvoker::addNativePath(const std::string& path)
629 if (!m_pythonPath.empty())
630 m_pythonPath += PY_PATH_SEP;
632 m_pythonPath += path;