00001
00002 #include <QString>
00003 #include <QCoreApplication>
00004 #include <QFile>
00005 #include <QDir>
00006
00007 #include "mythdirs.h"
00008 #include "mythcontext.h"
00009 #include "mythsystem.h"
00010 #include "exitcodes.h"
00011 #include "mythmiscutil.h"
00012 #include "mythlogging.h"
00013
00014 #include "netgrabbermanager.h"
00015 #include "netutils.h"
00016
00017 #define LOC QString("NetContent: ")
00018
00019 using namespace std;
00020
00021
00022
00023 GrabberScript::GrabberScript(const QString& title, const QString& image,
00024 const ArticleType &type, const QString& author,
00025 const bool& search, const bool& tree,
00026 const QString& description, const QString& commandline,
00027 const double& version) :
00028 MThread("GrabberScript"), m_lock(QMutex::Recursive)
00029 {
00030 m_title = title;
00031 m_image = image;
00032 m_type = type;
00033 m_author = author;
00034 m_search = search;
00035 m_tree = tree;
00036 m_description = description;
00037 m_commandline = commandline;
00038 m_version = version;
00039 }
00040
00041 GrabberScript::~GrabberScript()
00042 {
00043 wait();
00044 }
00045
00046 void GrabberScript::run()
00047 {
00048 RunProlog();
00049 QMutexLocker locker(&m_lock);
00050
00051 QString commandline = m_commandline;
00052 MythSystem getTree(commandline, QStringList("-T"),
00053 kMSRunShell | kMSStdOut | kMSBuffered);
00054 getTree.Run(900);
00055 uint status = getTree.Wait();
00056
00057 if( status == GENERIC_EXIT_CMD_NOT_FOUND )
00058 LOG(VB_GENERAL, LOG_ERR, LOC +
00059 QString("Internet Content Source %1 cannot run, file missing.")
00060 .arg(m_title));
00061 else if( status == GENERIC_EXIT_OK )
00062 {
00063 LOG(VB_GENERAL, LOG_INFO, LOC +
00064 QString("Internet Content Source %1 completed download, "
00065 "beginning processing...").arg(m_title));
00066
00067 QByteArray result = getTree.ReadAll();
00068
00069 QDomDocument domDoc;
00070 domDoc.setContent(result, true);
00071 QDomElement root = domDoc.documentElement();
00072 QDomElement channel = root.firstChildElement("channel");
00073
00074 clearTreeItems(m_title);
00075
00076 while (!channel.isNull())
00077 {
00078 parseDBTree(m_title, QString(), QString(), channel, GetType());
00079 channel = channel.nextSiblingElement("channel");
00080 }
00081 markTreeUpdated(this, QDateTime::currentDateTime());
00082 LOG(VB_GENERAL, LOG_INFO, LOC +
00083 QString("Internet Content Source %1 completed processing, "
00084 "marking as updated.").arg(m_title));
00085 }
00086 else
00087 LOG(VB_GENERAL, LOG_ERR, LOC +
00088 QString("Internet Content Source %1 crashed while grabbing tree.")
00089 .arg(m_title));
00090
00091 emit finished();
00092 RunEpilog();
00093 }
00094
00095 void GrabberScript::parseDBTree(const QString &feedtitle, const QString &path,
00096 const QString &pathThumb, QDomElement& domElem,
00097 const ArticleType &type)
00098 {
00099 QMutexLocker locker(&m_lock);
00100
00101 Parse parse;
00102 ResultItem::resultList articles;
00103
00104
00105 QDomElement fileitem = domElem.firstChildElement("item");
00106 while (!fileitem.isNull())
00107 {
00108 articles.append(parse.ParseItem(fileitem));
00109 fileitem = fileitem.nextSiblingElement("item");
00110 }
00111
00112 while (!articles.isEmpty())
00113 {
00114 insertTreeArticleInDB(feedtitle, path,
00115 pathThumb, articles.takeFirst(), type);
00116 }
00117
00118
00119 QDomElement diritem = domElem.firstChildElement("directory");
00120 while (!diritem.isNull())
00121 {
00122 QDomElement subfolder = diritem;
00123 QString dirname = diritem.attribute("name");
00124 QString dirthumb = diritem.attribute("thumbnail");
00125 dirname.replace("/", "|");
00126 QString pathToUse;
00127
00128 if (path.isEmpty())
00129 pathToUse = dirname;
00130 else
00131 pathToUse = QString("%1/%2").arg(path).arg(dirname);
00132
00133 parseDBTree(feedtitle,
00134 pathToUse,
00135 dirthumb,
00136 subfolder,
00137 type);
00138 diritem = diritem.nextSiblingElement("directory");
00139 }
00140 }
00141
00142 GrabberManager::GrabberManager() : m_lock(QMutex::Recursive)
00143 {
00144 m_updateFreq = (gCoreContext->GetNumSetting(
00145 "netsite.updateFreq", 24) * 3600 * 1000);
00146 m_timer = new QTimer();
00147 m_runningCount = 0;
00148 m_refreshAll = false;
00149 connect( m_timer, SIGNAL(timeout()),
00150 this, SLOT(timeout()));
00151 }
00152
00153 GrabberManager::~GrabberManager()
00154 {
00155 delete m_timer;
00156 }
00157
00158 void GrabberManager::startTimer()
00159 {
00160 m_timer->start(m_updateFreq);
00161 }
00162
00163 void GrabberManager::stopTimer()
00164 {
00165 m_timer->stop();
00166 }
00167
00168 void GrabberManager::doUpdate()
00169 {
00170 GrabberDownloadThread *gdt = new GrabberDownloadThread(this);
00171 if (m_refreshAll)
00172 gdt->refreshAll();
00173 gdt->start(QThread::LowPriority);
00174
00175 m_timer->start(m_updateFreq);
00176 }
00177
00178 void GrabberManager::timeout()
00179 {
00180 QMutexLocker locker(&m_lock);
00181 doUpdate();
00182 }
00183
00184 void GrabberManager::refreshAll()
00185 {
00186 m_refreshAll = true;
00187 }
00188
00189 GrabberDownloadThread::GrabberDownloadThread(QObject *parent) :
00190 MThread("GrabberDownload")
00191 {
00192 m_parent = parent;
00193 m_refreshAll = false;
00194 }
00195
00196 GrabberDownloadThread::~GrabberDownloadThread()
00197 {
00198 cancel();
00199 wait();
00200 }
00201
00202 void GrabberDownloadThread::cancel()
00203 {
00204 m_mutex.lock();
00205 qDeleteAll(m_scripts);
00206 m_scripts.clear();
00207 m_mutex.unlock();
00208 }
00209
00210 void GrabberDownloadThread::refreshAll()
00211 {
00212 m_mutex.lock();
00213 m_refreshAll = true;
00214 if (!isRunning())
00215 start();
00216 m_mutex.unlock();
00217 }
00218
00219 void GrabberDownloadThread::run()
00220 {
00221 RunProlog();
00222
00223 m_scripts = findAllDBTreeGrabbers();
00224 uint updateFreq = gCoreContext->GetNumSetting(
00225 "netsite.updateFreq", 24);
00226
00227 while (m_scripts.count())
00228 {
00229 GrabberScript *script = m_scripts.takeFirst();
00230 if (script && (needsUpdate(script, updateFreq) || m_refreshAll))
00231 {
00232 LOG(VB_GENERAL, LOG_INFO, LOC +
00233 QString("Internet Content Source %1 Updating...")
00234 .arg(script->GetTitle()));
00235 script->run();
00236 }
00237 delete script;
00238 }
00239 emit finished();
00240 if (m_parent)
00241 QCoreApplication::postEvent(m_parent, new GrabberUpdateEvent());
00242
00243 RunEpilog();
00244 }
00245
00246 Search::Search()
00247 : m_searchProcess(NULL)
00248 {
00249 m_videoList.clear();
00250 }
00251
00252 Search::~Search()
00253 {
00254 resetSearch();
00255
00256 delete m_searchProcess;
00257 m_searchProcess = NULL;
00258 }
00259
00260
00261 void Search::executeSearch(const QString &script, const QString &query, uint pagenum)
00262 {
00263 resetSearch();
00264
00265 LOG(VB_GENERAL, LOG_DEBUG, "Search::executeSearch");
00266 m_searchProcess = new MythSystem();
00267
00268 connect(m_searchProcess, SIGNAL(finished()),
00269 this, SLOT(slotProcessSearchExit()));
00270 connect(m_searchProcess, SIGNAL(error(uint)),
00271 this, SLOT(slotProcessSearchExit(uint)));
00272
00273 QString cmd = script;
00274
00275 QStringList args;
00276
00277 if (pagenum > 1)
00278 {
00279 args.append(QString("-p"));
00280 args.append(QString::number(pagenum));
00281 }
00282
00283 args.append("-S");
00284 QString term = query;
00285 args.append(ShellEscape(term));
00286
00287 LOG(VB_GENERAL, LOG_DEBUG, LOC +
00288 QString("Internet Search Query: %1 %2") .arg(cmd).arg(args.join(" ")));
00289
00290 uint flags = kMSRunShell | kMSStdOut | kMSBuffered | kMSRunBackground;
00291 m_searchProcess->SetCommand(cmd, args, flags);
00292 m_searchProcess->Run(40);
00293 }
00294
00295 void Search::resetSearch()
00296 {
00297 qDeleteAll(m_videoList);
00298 m_videoList.clear();
00299 }
00300
00301 void Search::process()
00302 {
00303 Parse parse;
00304 m_videoList = parse.parseRSS(m_document);
00305
00306 QDomNodeList entries = m_document.elementsByTagName("channel");
00307
00308 if (entries.count() == 0)
00309 {
00310 m_numResults = 0;
00311 m_numReturned = 0;
00312 m_numIndex = 0;
00313 return;
00314 }
00315
00316 QDomNode itemNode = entries.item(0);
00317
00318 QDomNode Node = itemNode.namedItem(QString("numresults"));
00319 if (!Node.isNull())
00320 {
00321 m_numResults = Node.toElement().text().toUInt();
00322 }
00323 else
00324 {
00325 QDomNodeList count = m_document.elementsByTagName("item");
00326
00327 if (count.count() == 0)
00328 m_numResults = 0;
00329 else
00330 m_numResults = count.count();
00331 }
00332
00333 Node = itemNode.namedItem(QString("returned"));
00334 if (!Node.isNull())
00335 {
00336 m_numReturned = Node.toElement().text().toUInt();
00337 }
00338 else
00339 {
00340 QDomNodeList entries = m_document.elementsByTagName("item");
00341
00342 if (entries.count() == 0)
00343 m_numReturned = 0;
00344 else
00345 m_numReturned = entries.count();
00346 }
00347
00348 Node = itemNode.namedItem(QString("startindex"));
00349 if (!Node.isNull())
00350 {
00351 m_numIndex = Node.toElement().text().toUInt();
00352 }
00353 else
00354 m_numIndex = 0;
00355
00356 }
00357
00358 void Search::slotProcessSearchExit(uint exitcode)
00359 {
00360 if (exitcode == GENERIC_EXIT_TIMEOUT)
00361 {
00362 LOG(VB_GENERAL, LOG_WARNING, LOC + "Internet Search Timeout");
00363
00364 if (m_searchProcess)
00365 {
00366 m_searchProcess->Term(true);
00367 m_searchProcess->deleteLater();
00368 m_searchProcess = NULL;
00369 }
00370 emit searchTimedOut(this);
00371 return;
00372 }
00373
00374 if (exitcode != GENERIC_EXIT_OK)
00375 {
00376 m_document.setContent(QString());
00377 }
00378 else
00379 {
00380 LOG(VB_GENERAL, LOG_INFO, LOC +
00381 "Internet Search Successfully Completed");
00382
00383 m_data = m_searchProcess->ReadAll();
00384 m_document.setContent(m_data, true);
00385 }
00386
00387 m_searchProcess->deleteLater();
00388 m_searchProcess = NULL;
00389 emit finishedSearch(this);
00390 }
00391
00392 void Search::SetData(QByteArray data)
00393 {
00394 m_data = data;
00395 m_document.setContent(m_data, true);
00396
00397 }
00398
