MythTV: backendutil.cpp Source File

00001 #include <cstdlib> // for llabs
00002 
00003 #include "mythconfig.h"
00004 #if defined(CONFIG_DARWIN) || defined(__FreeBSD__)
00005 #include <sys/param.h>
00006 #include <sys/mount.h>
00007 #elif __linux__
00008 #include <sys/vfs.h>
00009 #endif
00010 
00011 #include <qdir.h>
00012 #include <qmutex.h>
00013 #include <qmap.h>
00014 
00015 #include "backendutil.h"
00016 #include "remoteutil.h"
00017 
00018 #include "libmyth/mythcontext.h"
00019 #include "libmyth/mythdbcon.h"
00020 #include "libmyth/util.h"
00021 #include "libmyth/compat.h"
00022 
00023 static QMap<QString, int> fsID_cache;
00024 static QMutex cache_lock;
00025 
00027 static int GetfsID(vector<FileSystemInfo>::iterator fsInfo)
00028 {
00029     QString fskey = fsInfo->hostname + ":" + fsInfo->directory;
00030     QMutexLocker lock(&cache_lock);
00031     if (!fsID_cache.contains(fskey))
00032         fsID_cache[fskey] = fsID_cache.count();
00033 
00034     return fsID_cache[fskey];
00035 }
00036 
00037 static size_t GetCurrentMaxBitrate(QMap<int, EncoderLink *> *encoderList)
00038 {
00039     size_t totalKBperMin = 0;
00040 
00041     QMap<int, EncoderLink*>::iterator it = encoderList->begin();
00042     for (; it != encoderList->end(); ++it)
00043     {
00044         EncoderLink *enc = *it;
00045 
00046         if (!enc->IsConnected() || !enc->IsBusy())
00047             continue;
00048 
00049         long long maxBitrate = enc->GetMaxBitrate();
00050         if (maxBitrate<=0)
00051             maxBitrate = 19500000LL;
00052         long long thisKBperMin = (((size_t)maxBitrate)*((size_t)15))>>11;
00053         totalKBperMin += thisKBperMin;
00054         VERBOSE(VB_FILE, QString("Cardid %1: max bitrate %2 KB/min")
00055                 .arg(enc->GetCardID()).arg(thisKBperMin));
00056     }
00057 
00058     VERBOSE(VB_FILE, QString("Maximal bitrate of busy encoders is %1 KB/min")
00059             .arg(totalKBperMin));
00060 
00061     return totalKBperMin;
00062 }
00063 
00064 void BackendQueryDiskSpace(QStringList &strlist,
00065                            QMap <int, EncoderLink *> *encoderList,
00066                            bool consolidated, bool allHosts)
00067 {
00068     QString allHostList = gContext->GetHostName();
00069     long long totalKB = -1, usedKB = -1;
00070     QMap <QString, bool>foundDirs;
00071     QString driveKey;
00072     QString localStr = "1";
00073     struct statfs statbuf;
00074     MSqlQuery query(MSqlQuery::InitCon());
00075     query.prepare("SELECT MIN(id),dirname "
00076                   "FROM storagegroup "
00077                   "WHERE hostname = :HOSTNAME "
00078                   "GROUP BY dirname;");
00079     query.bindValue(":HOSTNAME", gContext->GetHostName());
00080 
00081     if (query.exec() && query.isActive())
00082     {
00083         // If we don't have any dirs of our own, fallback to list of Default
00084         // dirs since that is what StorageGroup::Init() does.
00085         if (!query.size())
00086         {
00087             query.prepare("SELECT MIN(id),dirname "
00088                           "FROM storagegroup "
00089                           "WHERE groupname = :GROUP "
00090                           "GROUP BY dirname;");
00091             query.bindValue(":GROUP", "Default");
00092             query.exec();
00093         }
00094 
00095         QDir checkDir("");
00096         QString dirID;
00097         QString currentDir;
00098         while (query.next())
00099         {
00100             dirID = query.value(0).toString();
00101             currentDir = query.value(1).toString();
00102             if (currentDir.right(1) == "/")
00103                 currentDir.remove(currentDir.length() - 1, 1);
00104 
00105             checkDir.setPath(currentDir);
00106             if (!foundDirs.contains(currentDir))
00107             {
00108                 if (checkDir.exists())
00109                 {
00110                     getDiskSpace(currentDir.ascii(), totalKB, usedKB);
00111                     bzero(&statbuf, sizeof(statbuf));
00112                     localStr = "1"; // Assume local
00113 
00114 #ifdef CONFIG_DARWIN
00115                     // FIXME, Not sure what these strings should be so this
00116                     // code is really a placeholder for now.
00117                     if ((statfs(currentDir, &statbuf) == 0) &&
00118                         ((!strcmp(statbuf.f_fstypename, "nfs")) ||   // NFS|FTP
00119                          (!strcmp(statbuf.f_fstypename, "afpfs")) || // ApplShr
00120                          (!strcmp(statbuf.f_fstypename, "smbfs"))))  // SMB
00121                         localStr = "0";
00122 #elif __linux__
00123                     if ((statfs(currentDir, &statbuf) == 0) &&
00124                         ((statbuf.f_type == 0x6969) ||     // NFS
00125                          (statbuf.f_type == 0x517B) ||     // SMB
00126                          (statbuf.f_type == (long)0xFF534D42)))  // CIFS
00127                         localStr = "0";
00128 #endif
00129                     strlist << gContext->GetHostName();
00130                     strlist << currentDir;
00131                     strlist << localStr;
00132                     strlist << "-1"; // Ignore fsID
00133                     strlist << dirID;
00134                     encodeLongLong(strlist, totalKB);
00135                     encodeLongLong(strlist, usedKB);
00136 
00137                     foundDirs[currentDir] = true;
00138                 }
00139                 else
00140                     foundDirs[currentDir] = false;
00141             }
00142         }
00143     }
00144 
00145     if (allHosts)
00146     {
00147         QMap <QString, bool> backendsCounted;
00148         QString encoderHost;
00149         QMap<int, EncoderLink *>::Iterator eit = encoderList->begin();
00150         while (eit != encoderList->end())
00151         {
00152             encoderHost = eit.data()->GetHostName();
00153             if (eit.data()->IsConnected() &&
00154                 !eit.data()->IsLocal() &&
00155                 !backendsCounted.contains(encoderHost))
00156             {
00157                 backendsCounted[encoderHost] = true;
00158 
00159                 eit.data()->GetDiskSpace(strlist);
00160 
00161                 allHostList += "," + encoderHost;
00162             }
00163             ++eit;
00164         }
00165     }
00166 
00167     if (!consolidated)
00168         return;
00169 
00170     FileSystemInfo fsInfo;
00171     vector<FileSystemInfo> fsInfos;
00172 
00173     QStringList::const_iterator it = strlist.begin();
00174     while (it != strlist.end())
00175     {
00176         fsInfo.hostname = *(it++);
00177         fsInfo.directory = *(it++);
00178         fsInfo.isLocal = (*(it++)).toInt();
00179         fsInfo.fsID = (*(it++)).toInt();
00180         fsInfo.dirID = (*(it++)).toInt();
00181         fsInfo.totalSpaceKB = decodeLongLong(strlist, it);
00182         fsInfo.usedSpaceKB = decodeLongLong(strlist, it);
00183         fsInfo.freeSpaceKB = fsInfo.totalSpaceKB - fsInfo.usedSpaceKB;
00184         fsInfos.push_back(fsInfo);
00185     }
00186     strlist.clear();
00187 
00188     // Consolidate hosts sharing storage
00189     size_t maxWriteFiveSec = GetCurrentMaxBitrate(encoderList)/12 /*5 seconds*/;
00190     maxWriteFiveSec = max((size_t)2048, maxWriteFiveSec); // safety for NFS mounted dirs
00191     vector<FileSystemInfo>::iterator it1, it2;
00192     for (it1 = fsInfos.begin(); it1 != fsInfos.end(); it1++)
00193     {
00194         if (it1->fsID == -1)
00195         {
00196             it1->fsID = GetfsID(it1);
00197             it1->directory =
00198                 it1->hostname.section(".", 0, 0) + ":" + it1->directory;
00199         }
00200 
00201         it2 = it1;
00202         for (it2++; it2 != fsInfos.end(); it2++)
00203         {
00204             // Sometimes the space reported for an NFS mounted dir is slightly
00205             // different than when it is locally mounted because of block sizes
00206             if (it2->fsID == -1 &&
00207                 (absLongLong(it1->totalSpaceKB - it2->totalSpaceKB) <= 1024) &&
00208                 ((size_t)absLongLong(it1->usedSpaceKB - it2->usedSpaceKB)
00209                  <= maxWriteFiveSec))
00210             {
00211                 if (!it1->hostname.contains(it2->hostname))
00212                     it1->hostname = it1->hostname + "," + it2->hostname;
00213                 it1->directory = it1->directory + "," +
00214                     it2->hostname.section(".", 0, 0) + ":" + it2->directory;
00215                 fsInfos.erase(it2);
00216                 it2 = it1;
00217             }
00218         }
00219     }
00220 
00221     // Passed the cleaned list back
00222     totalKB = 0;
00223     usedKB  = 0;
00224     for (it1 = fsInfos.begin(); it1 != fsInfos.end(); it1++)
00225     {
00226         strlist << it1->hostname;
00227         strlist << it1->directory;
00228         strlist << QString::number(it1->isLocal);
00229         strlist << QString::number(it1->fsID);
00230         strlist << QString::number(it1->dirID);
00231         encodeLongLong(strlist, it1->totalSpaceKB);
00232         encodeLongLong(strlist, it1->usedSpaceKB);
00233 
00234         totalKB += it1->totalSpaceKB;
00235         usedKB  += it1->usedSpaceKB;
00236     }
00237 
00238     if (allHosts)
00239     {
00240         strlist << allHostList;
00241         strlist << "TotalDiskSpace";
00242         strlist << "0";
00243         strlist << "-2";
00244         strlist << "-2";
00245         encodeLongLong(strlist, totalKB);
00246         encodeLongLong(strlist, usedKB);
00247     }
00248 }
00249 
00250 void GetFilesystemInfos(QMap<int, EncoderLink*> *tvList,
00251                         vector <FileSystemInfo> &fsInfos)
00252 {
00253     QStringList strlist;
00254     FileSystemInfo fsInfo;
00255 
00256     fsInfos.clear();
00257 
00258     BackendQueryDiskSpace(strlist, tvList, false, true);
00259 
00260     QStringList::const_iterator it = strlist.begin();
00261     while (it != strlist.end())
00262     {
00263         fsInfo.hostname = *(it++);
00264         fsInfo.directory = *(it++);
00265         fsInfo.isLocal = (*(it++)).toInt();
00266         fsInfo.fsID = -1;
00267         it++;
00268         fsInfo.dirID = (*(it++)).toInt();
00269         fsInfo.totalSpaceKB = decodeLongLong(strlist, it);
00270         fsInfo.liveTVSpaceKB = 0; // FIXME, placeholder
00271         fsInfo.usedSpaceKB = decodeLongLong(strlist, it);
00272         fsInfo.freeSpaceKB = fsInfo.totalSpaceKB - fsInfo.usedSpaceKB;
00273         fsInfo.weight = 0;
00274         fsInfos.push_back(fsInfo);
00275     }
00276 
00277     VERBOSE(VB_SCHEDULE+VB_FILE, "Determining unique filesystems");
00278     size_t maxWriteFiveSec = GetCurrentMaxBitrate(tvList)/12  /*5 seconds*/;
00279     maxWriteFiveSec = max((size_t)2048, maxWriteFiveSec); // safety for NFS mounted dirs
00280     vector<FileSystemInfo>::iterator it1, it2;
00281     for (it1 = fsInfos.begin(); it1 != fsInfos.end(); it1++)
00282     {
00283         if (it1->fsID == -1)
00284             it1->fsID = GetfsID(it1);
00285         else
00286             continue;
00287 
00288         VERBOSE(VB_SCHEDULE+VB_FILE,
00289             QString("%1:%2 (fsID %3, dirID %4) using %5 out of %6 KB, "
00290                     "looking for matches")
00291                     .arg(it1->hostname).arg(it1->directory)
00292                     .arg(it1->fsID).arg(it1->dirID)
00293                     .arg(it1->usedSpaceKB).arg(it1->totalSpaceKB));
00294         it2 = it1;
00295         for (it2++; it2 != fsInfos.end(); it2++)
00296         {
00297             VERBOSE(VB_SCHEDULE+VB_FILE,
00298                 QString("    Checking %1:%2 (dirID %3) using %4 of %5 KB")
00299                         .arg(it2->hostname).arg(it2->directory).arg(it2->dirID)
00300                         .arg(it2->usedSpaceKB).arg(it2->totalSpaceKB));
00301             VERBOSE(VB_SCHEDULE+VB_FILE,
00302                 QString("        Total KB Diff: %1 (want <= 1024)") 
00303                 .arg((long)absLongLong(it1->totalSpaceKB - it2->totalSpaceKB)));
00304             VERBOSE(VB_SCHEDULE+VB_FILE,
00305                 QString("        Used  KB Diff: %1 (want <= %2)")
00306                 .arg((size_t)absLongLong(it1->usedSpaceKB - it2->usedSpaceKB))
00307                 .arg(maxWriteFiveSec));
00308 
00309             // Sometimes the space reported for an NFS mounted dir is slightly
00310             // different than when it is locally mounted because of block sizes
00311             if (it2->fsID == -1 &&
00312                 (absLongLong(it1->totalSpaceKB - it2->totalSpaceKB) <= 1024) &&
00313                 ((size_t)absLongLong(it1->usedSpaceKB - it2->usedSpaceKB)
00314                  <= maxWriteFiveSec))
00315             {
00316                 it2->fsID = it1->fsID;
00317 
00318                 VERBOSE(VB_SCHEDULE+VB_FILE,
00319                     QString("    MATCH Found: %1:%2 will use fsID %3")
00320                             .arg(it2->hostname).arg(it2->directory)
00321                             .arg(it2->fsID));
00322             }
00323         }
00324     }
00325 
00326     if (print_verbose_messages & (VB_FILE|VB_SCHEDULE))
00327     {
00328         cout << "--- GetFilesystemInfos directory list start ---" << endl;
00329         for (it1 = fsInfos.begin(); it1 != fsInfos.end(); it1++)
00330         {
00331             cout << "Dir: " << it1->hostname << ":" << it1->directory << endl;
00332             cout << "     Location: ";
00333             if (it1->isLocal)
00334                 cout << "Local";
00335             else
00336                 cout << "Remote";
00337             cout << endl;
00338             cout << "     fsID    : " << it1->fsID << endl;
00339             cout << "     dirID   : " << it1->dirID << endl;
00340             cout << "     TotalKB : " << it1->totalSpaceKB << endl;
00341             cout << "     UsedKB  : " << it1->usedSpaceKB << endl;
00342             cout << "     FreeKB  : " << it1->freeSpaceKB << endl;
00343             cout << endl;
00344         }
00345         cout << "--- GetFilesystemInfos directory list end ---" << endl;
00346     }
00347 }
00348 
00349 QMutex recordingPathLock;
00350 QMap <QString, QString> recordingPathCache;
00351 
00352 QString GetPlaybackURL(ProgramInfo *pginfo, bool storePath)
00353 {
00354     QString result = "";
00355     QMutexLocker locker(&recordingPathLock);
00356     QString cacheKey = QString("%1:%2").arg(pginfo->chanid)
00357                                .arg(pginfo->recstartts.toString(Qt::ISODate));
00358     if ((recordingPathCache.contains(cacheKey)) &&
00359         (QFile::exists(recordingPathCache[cacheKey])))
00360     {
00361         result = recordingPathCache[cacheKey];
00362         if (!storePath)
00363             recordingPathCache.remove(cacheKey);
00364     }
00365     else
00366     {
00367         result = pginfo->GetPlaybackURL(false, true);
00368         if (storePath && result.left(1) == "/")
00369             recordingPathCache[cacheKey] = result;
00370     }
00371 
00372     return result;
00373 }
00374 
00375 /* vim: set expandtab tabstop=4 shiftwidth=4: */