Rename and restructure the baseline (lancelot) testing code

In preparation for addition of new baseline tests, establish a new test category, "baseline". This is similar to the category "benchmarks" in that it contains tests that use the QTest framework, but conceptually are not unit tests, in contrast to those under auto/. Move the existing QPainter baseline test, tst_lancelot, into this new category, and rename it accordingly. Baseline tests use the QBaselineTest extension to QTest. Move that extension too into the tests/baseline directory, allowing the clean out of the baselineserver directory. Pick-to: 6.2 Change-Id: I1b527f5867c953b1d22be73798fcf7d1494712ea Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org> Reviewed-by: Eirik Aavitsland <eirik.aavitsland@qt.io> Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>
author: Eirik Aavitsland <eirik.aavitsland@qt.io> 2021-11-15 13:36:48 +0100
committer: Volker Hilsheimer <volker.hilsheimer@qt.io> 2021-11-16 14:01:50 +0100
commit: e8f93e38debf4f46dbda4de198b9dd475df42d5c (patch)
tree: c042e1fea11d2b63788f1b8855030ecd96e13c19 /tests/baselineserver/src/baselineserver.cpp
parent: 9a131b59ee561e300fb6078886a85dd7b3c53599 (diff)
1 files changed, 0 insertions, 853 deletions
diff --git a/tests/baselineserver/src/baselineserver.cpp b/tests/baselineserver/src/baselineserver.cpp
deleted file mode 100644
index c6e12dc09cf..00000000000
--- a/tests/baselineserver/src/baselineserver.cpp
+++ /dev/null
@@ -1,853 +0,0 @@
-/****************************************************************************
-**
-** Copyright (C) 2016 The Qt Company Ltd.
-** Contact: https://www.qt.io/licensing/
-**
-** This file is part of the test suite of the Qt Toolkit.
-**
-** $QT_BEGIN_LICENSE:GPL-EXCEPT$
-** Commercial License Usage
-** Licensees holding valid commercial Qt licenses may use this file in
-** accordance with the commercial license agreement provided with the
-** Software or, alternatively, in accordance with the terms contained in
-** a written agreement between you and The Qt Company. For licensing terms
-** and conditions see https://www.qt.io/terms-conditions. For further
-** information use the contact form at https://www.qt.io/contact-us.
-**
-** GNU General Public License Usage
-** Alternatively, this file may be used under the terms of the GNU
-** General Public License version 3 as published by the Free Software
-** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
-** included in the packaging of this file. Please review the following
-** information to ensure the GNU General Public License requirements will
-** be met: https://www.gnu.org/licenses/gpl-3.0.html.
-**
-** $QT_END_LICENSE$
-**
-****************************************************************************/
-
-#define QT_USE_FAST_CONCATENATION
-#define QT_USE_FAST_OPERATOR_PLUS
-
-#include "baselineserver.h"
-#include <QBuffer>
-#include <QFile>
-#include <QDir>
-#include <QCoreApplication>
-#include <QFileInfo>
-#include <QHostInfo>
-#include <QTextStream>
-#include <QProcess>
-#include <QDirIterator>
-#include <QUrl>
-#include <QRegularExpression>
-
-// extra fields, for use in image metadata storage
-const QString PI_ImageChecksum(QLS("ImageChecksum"));
-const QString PI_RunId(QLS("RunId"));
-const QString PI_CreationDate(QLS("CreationDate"));
-
-QString BaselineServer::storage;
-QString BaselineServer::url;
-QStringList BaselineServer::pathKeys;
-
-BaselineServer::BaselineServer(QObject *parent)
-    : QTcpServer(parent), lastRunIdIdx(0)
-{
-    QFileInfo me(QCoreApplication::applicationFilePath());
-    meLastMod = me.lastModified();
-    heartbeatTimer = new QTimer(this);
-    connect(heartbeatTimer, SIGNAL(timeout()), this, SLOT(heartbeat()));
-    heartbeatTimer->start(HEARTBEAT*1000);
-}
-
-QString BaselineServer::storagePath()
-{
-    if (storage.isEmpty()) {
-        storage = QLS(qgetenv("QT_LANCELOT_DIR"));
-        if (storage.isEmpty())
-            storage = QLS("/var/www");
-    }
-    return storage;
-}
-
-QString BaselineServer::baseUrl()
-{
-    if (url.isEmpty()) {
-        url = QLS("http://")
-                + QHostInfo::localHostName().toLatin1() + '.'
-                + QHostInfo::localDomainName().toLatin1() + '/';
-    }
-    return url;
-}
-
-QStringList BaselineServer::defaultPathKeys()
-{
-    if (pathKeys.isEmpty())
-        pathKeys << PI_QtVersion << PI_QMakeSpec << PI_HostName;
-    return pathKeys;
-}
-
-void BaselineServer::incomingConnection(qintptr socketDescriptor)
-{
-    QString runId = QDateTime::currentDateTime().toString(QLS("MMMdd-hhmmss"));
-    if (runId == lastRunId) {
-        runId += QLC('-') + QString::number(++lastRunIdIdx);
-    } else {
-        lastRunId = runId;
-        lastRunIdIdx = 0;
-    }
-    qDebug() << "Server: New connection! RunId:" << runId;
-    BaselineThread *thread = new BaselineThread(runId, socketDescriptor, this);
-    connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater()));
-    thread->start();
-}
-
-void BaselineServer::heartbeat()
-{
-    // The idea is to exit to be restarted when modified, as soon as not actually serving
-    QFileInfo me(QCoreApplication::applicationFilePath());
-    if (me.lastModified() == meLastMod)
-        return;
-    if (!me.exists() || !me.isExecutable())
-        return;
-
-    //# (could close() here to avoid accepting new connections, to avoid livelock)
-    //# also, could check for a timeout to force exit, to avoid hung threads blocking
-    bool isServing = false;
-    foreach(BaselineThread *thread, findChildren<BaselineThread *>()) {
-        if (thread->isRunning()) {
-            isServing = true;
-            break;
-        }
-    }
-
-    if (!isServing)
-        QCoreApplication::exit();
-}
-
-BaselineThread::BaselineThread(const QString &runId, int socketDescriptor, QObject *parent)
-    : QThread(parent), runId(runId), socketDescriptor(socketDescriptor)
-{
-}
-
-void BaselineThread::run()
-{
-    BaselineHandler handler(runId, socketDescriptor);
-    exec();
-}
-
-
-BaselineHandler::BaselineHandler(const QString &runId, int socketDescriptor)
-    : QObject(), runId(runId), connectionEstablished(false), settings(0), fuzzLevel(0)
-{
-    idleTimer = new QTimer(this);
-    idleTimer->setSingleShot(true);
-    idleTimer->setInterval(IDLE_CLIENT_TIMEOUT * 1000);
-    connect(idleTimer, SIGNAL(timeout()), this, SLOT(idleClientTimeout()));
-    idleTimer->start();
-
-    if (socketDescriptor == -1)
-        return;
-
-    connect(&proto.socket, SIGNAL(readyRead()), this, SLOT(receiveRequest()));
-    connect(&proto.socket, SIGNAL(disconnected()), this, SLOT(receiveDisconnect()));
-    proto.socket.setSocketDescriptor(socketDescriptor);
-    proto.socket.setSocketOption(QAbstractSocket::KeepAliveOption, 1);
-}
-
-const char *BaselineHandler::logtime()
-{
-    return 0;
-    //return QTime::currentTime().toString(QLS("mm:ss.zzz"));
-}
-
-QString BaselineHandler::projectPath(bool absolute) const
-{
-    QString p = clientInfo.value(PI_Project);
-    return absolute ? BaselineServer::storagePath() + QLC('/') + p : p;
-}
-
-bool BaselineHandler::checkClient(QByteArray *errMsg, bool *dryRunMode)
-{
-    if (!errMsg)
-        return false;
-    if (clientInfo.value(PI_Project).isEmpty() || clientInfo.value(PI_TestCase).isEmpty()) {
-        *errMsg = "No Project and/or TestCase specified in client info.";
-        return false;
-    }
-
-    // Determine ad-hoc state ### hardcoded for now
-    if (clientInfo.value(PI_TestCase) == QLS("tst_Lancelot")) {
-        //### Todo: push this stuff out in a script
-        if (!clientInfo.isAdHocRun()) {
-            // ### comp. with earlier versions still running (4.8) (?)
-            clientInfo.setAdHocRun(clientInfo.value(PI_PulseGitBranch).isEmpty() && clientInfo.value(PI_PulseTestrBranch).isEmpty());
-        }
-    }
-    else {
-        // TBD
-    }
-
-    if (clientInfo.isAdHocRun()) {
-        if (dryRunMode)
-            *dryRunMode = false;
-        return true;
-    }
-
-    // Not ad hoc: filter the client
-    settings->beginGroup("ClientFilters");
-    bool matched = false;
-    bool dryRunReq = false;
-    foreach (const QString &rule, settings->childKeys()) {
-        //qDebug() << "  > RULE" << rule;
-        dryRunReq = false;
-        QString ruleMode = settings->value(rule).toString().toLower();
-        if (ruleMode == QLS("dryrun"))
-            dryRunReq = true;
-        else if (ruleMode != QLS("enabled"))
-            continue;
-        settings->beginGroup(rule);
-        bool ruleMatched = true;
-        foreach (const QString &filterKey, settings->childKeys()) {
-            //qDebug() << "    > FILTER" << filterKey;
-            QString filter = settings->value(filterKey).toString();
-            if (filter.isEmpty())
-                continue;
-            QString platVal = clientInfo.value(filterKey);
-            if (!platVal.contains(filter)) {
-                ruleMatched = false;
-                break;
-            }
-        }
-        if (ruleMatched) {
-            ruleName = rule;
-            matched = true;
-            break;
-        }
-        settings->endGroup();
-    }
-
-    if (!matched && errMsg)
-        *errMsg = "Non-adhoc client did not match any filter rule in " + settings->fileName().toLatin1();
-
-    if (matched && dryRunMode)
-        *dryRunMode = dryRunReq;
-
-    // NB! Must reset the settings object before returning
-    while (!settings->group().isEmpty())
-        settings->endGroup();
-
-    return matched;
-}
-
-bool BaselineHandler::establishConnection()
-{
-    if (!proto.acceptConnection(&clientInfo)) {
-        qWarning() << runId << logtime() << "Accepting new connection from" << proto.socket.peerAddress().toString() << "failed." << proto.errorMessage();
-        proto.sendBlock(BaselineProtocol::Abort, proto.errorMessage().toLatin1());  // In case the client can hear us, tell it what's wrong.
-        proto.socket.disconnectFromHost();
-        return false;
-    }
-    QString logMsg;
-    foreach (QString key, clientInfo.keys()) {
-        if (key != PI_HostName && key != PI_HostAddress)
-            logMsg += key + QLS(": '") + clientInfo.value(key) + QLS("', ");
-    }
-    qDebug() << runId << logtime() << "Connection established with" << clientInfo.value(PI_HostName)
-             << '[' << qPrintable(clientInfo.value(PI_HostAddress)) << ']' << logMsg
-             << "Overrides:" << clientInfo.overrides() << "AdHoc-Run:" << clientInfo.isAdHocRun();
-
-    // ### Hardcoded backwards compatibility: add project field for certain existing clients that lack it
-    if (clientInfo.value(PI_Project).isEmpty()) {
-        QString tc = clientInfo.value(PI_TestCase);
-        if (tc == QLS("tst_Lancelot"))
-            clientInfo.insert(PI_Project, QLS("Raster"));
-        else if (tc == QLS("tst_Scenegraph"))
-            clientInfo.insert(PI_Project, QLS("SceneGraph"));
-        else
-            clientInfo.insert(PI_Project, QLS("Other"));
-    }
-
-    QString settingsFile = projectPath() + QLS("/config.ini");
-    settings = new QSettings(settingsFile, QSettings::IniFormat, this);
-
-    QByteArray errMsg;
-    bool dryRunMode = false;
-    if (!checkClient(&errMsg, &dryRunMode)) {
-        qDebug() << runId << logtime() << "Rejecting connection:" << errMsg;
-        proto.sendBlock(BaselineProtocol::Abort, errMsg);
-        proto.socket.disconnectFromHost();
-        return false;
-    }
-
-    fuzzLevel = qBound(0, settings->value("FuzzLevel").toInt(), 100);
-    if (!clientInfo.isAdHocRun()) {
-        qDebug() << runId << logtime() << "Client matches filter rule" << ruleName
-                 << "Dryrun:" << dryRunMode
-                 << "FuzzLevel:" << fuzzLevel
-                 << "ReportMissingResults:" << settings->value("ReportMissingResults").toBool();
-    }
-
-    proto.sendBlock(dryRunMode ? BaselineProtocol::DoDryRun : BaselineProtocol::Ack, QByteArray());
-    report.init(this, runId, clientInfo, settings);
-    return true;
-}
-
-void BaselineHandler::receiveRequest()
-{
-    idleTimer->start();   // Restart idle client timeout
-
-    if (!connectionEstablished) {
-        connectionEstablished = establishConnection();
-        return;
-    }
-
-    QByteArray block;
-    BaselineProtocol::Command cmd;
-    if (!proto.receiveBlock(&cmd, &block)) {
-        qWarning() << runId << logtime() << "Command reception failed. "<< proto.errorMessage();
-        QThread::currentThread()->exit(1);
-        return;
-    }
-
-    switch(cmd) {
-    case BaselineProtocol::RequestBaselineChecksums:
-        provideBaselineChecksums(block);
-        break;
-    case BaselineProtocol::AcceptMatch:
-        recordMatch(block);
-        break;
-    case BaselineProtocol::AcceptNewBaseline:
-        storeImage(block, true);
-        break;
-    case BaselineProtocol::AcceptMismatch:
-        storeImage(block, false);
-        break;
-    default:
-        qWarning() << runId << logtime() << "Unknown command received. " << proto.errorMessage();
-        proto.sendBlock(BaselineProtocol::UnknownError, QByteArray());
-    }
-}
-
-
-void BaselineHandler::provideBaselineChecksums(const QByteArray &itemListBlock)
-{
-    ImageItemList itemList;
-    QDataStream ds(itemListBlock);
-    ds >> itemList;
-    qDebug() << runId << logtime() << "Received request for checksums for" << itemList.count()
-             << "items in test function" << itemList.at(0).testFunction;
-
-    for (ImageItemList::iterator i = itemList.begin(); i != itemList.end(); ++i) {
-        i->imageChecksums.clear();
-        i->status = ImageItem::BaselineNotFound;
-        QString prefix = pathForItem(*i, true);
-        PlatformInfo itemData = fetchItemMetadata(prefix);
-        if (itemData.contains(PI_ImageChecksum)) {
-            bool ok = false;
-            quint64 checksum = itemData.value(PI_ImageChecksum).toULongLong(&ok, 16);
-            if (ok) {
-                i->imageChecksums.prepend(checksum);
-                i->status = ImageItem::Ok;
-            }
-        }
-    }
-
-    // Find and mark blacklisted items
-    QString context = pathForItem(itemList.at(0), true, false).section(QLC('/'), 0, -2);
-    if (itemList.count() > 0) {
-        QFile file(BaselineServer::storagePath() + QLC('/') + context + QLS("/BLACKLIST"));
-        if (file.open(QIODevice::ReadOnly)) {
-            QTextStream in(&file);
-            do {
-                QString itemName = in.readLine();
-                if (!itemName.isNull()) {
-                    for (ImageItemList::iterator i = itemList.begin(); i != itemList.end(); ++i) {
-                        if (i->itemName == itemName)
-                            i->status = ImageItem::IgnoreItem;
-                    }
-                }
-            } while (!in.atEnd());
-        }
-    }
-
-    QByteArray block;
-    QDataStream ods(&block, QIODevice::WriteOnly);
-    ods << itemList;
-    proto.sendBlock(BaselineProtocol::Ack, block);
-    report.addItems(itemList);
-}
-
-
-void BaselineHandler::recordMatch(const QByteArray &itemBlock)
-{
-    QDataStream ds(itemBlock);
-    ImageItem item;
-    ds >> item;
-    report.addResult(item);
-    proto.sendBlock(BaselineProtocol::Ack, QByteArray());
-}
-
-
-void BaselineHandler::storeImage(const QByteArray &itemBlock, bool isBaseline)
-{
-    QDataStream ds(itemBlock);
-    ImageItem item;
-    ds >> item;
-
-    if (isBaseline && !clientInfo.overrides().isEmpty()) {
-        qDebug() << runId << logtime() << "Received baseline from client with override info, ignoring. Item:" << item.itemName;
-        proto.sendBlock(BaselineProtocol::UnknownError, "New baselines not accepted from client with override info.");
-        return;
-    }
-
-    QString blPrefix = pathForItem(item, true);
-    QString mmPrefix = pathForItem(item, false);
-    QString prefix = isBaseline ? blPrefix : mmPrefix;
-
-    qDebug() << runId << logtime() << "Received" << (isBaseline ? "baseline" : "mismatched") << "image for:" << item.itemName << "Storing in" << prefix;
-
-    // Reply to the client
-    QString msg;
-    if (isBaseline)
-        msg = QLS("New baseline image stored: ") + blPrefix + QLS(FileFormat);
-    else
-        msg = BaselineServer::baseUrl() + report.filePath();
-
-    if (isBaseline || !fuzzLevel)
-        proto.sendBlock(BaselineProtocol::Ack, msg.toLatin1());  // Do early reply if possible: don't make the client wait longer than necessary
-
-    // Store the image
-    QString dir = prefix.section(QLC('/'), 0, -2);
-    QDir cwd;
-    if (!cwd.exists(dir))
-        cwd.mkpath(dir);
-    item.image.save(prefix + QLS(FileFormat), FileFormat);
-
-    PlatformInfo itemData = clientInfo;
-    itemData.insert(PI_ImageChecksum, QString::number(item.imageChecksums.at(0), 16));  //# Only the first is stored. TBD: get rid of list
-    itemData.insert(PI_RunId, runId);
-    itemData.insert(PI_CreationDate, QDateTime::currentDateTime().toString());
-    storeItemMetadata(itemData, prefix);
-
-    if (!isBaseline) {
-        // Do fuzzy matching
-        bool fuzzyMatch = false;
-        if (fuzzLevel) {
-            BaselineProtocol::Command cmd = BaselineProtocol::Ack;
-            fuzzyMatch = fuzzyCompare(blPrefix, mmPrefix);
-            if (fuzzyMatch) {
-                msg.prepend(QString("Fuzzy match at fuzzlevel %1%. Report: ").arg(fuzzLevel));
-                cmd = BaselineProtocol::FuzzyMatch;
-            }
-            proto.sendBlock(cmd, msg.toLatin1());  // We didn't reply earlier
-        }
-
-        // Add to report
-        item.status = fuzzyMatch ? ImageItem::FuzzyMatch : ImageItem::Mismatch;
-        report.addResult(item);
-    }
-}
-
-
-void BaselineHandler::storeItemMetadata(const PlatformInfo &metadata, const QString &path)
-{
-    QFile file(path + QLS(MetadataFileExt));
-    if (!file.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
-        qWarning() << runId << logtime() << "ERROR: could not write to file" << file.fileName();
-        return;
-    }
-    QTextStream out(&file);
-    PlatformInfo::const_iterator it = metadata.constBegin();
-    while (it != metadata.constEnd()) {
-        out << it.key()  << ": " << it.value() << endl;
-        ++it;
-    }
-    file.close();
-}
-
-
-PlatformInfo BaselineHandler::fetchItemMetadata(const QString &path)
-{
-    PlatformInfo res;
-    QFile file(path + QLS(MetadataFileExt));
-    if (!file.open(QIODevice::ReadOnly) || !QFile::exists(path + QLS(FileFormat)))
-        return res;
-    QTextStream in(&file);
-    do {
-        QString line = in.readLine();
-        int idx = line.indexOf(QLS(": "));
-        if (idx > 0)
-            res.insert(line.left(idx), line.mid(idx+2));
-    } while (!in.atEnd());
-    return res;
-}
-
-
-void BaselineHandler::idleClientTimeout()
-{
-    qWarning() << runId << logtime() << "Idle client timeout: no request received for" << IDLE_CLIENT_TIMEOUT << "seconds, terminating connection.";
-    proto.socket.disconnectFromHost();
-}
-
-
-void BaselineHandler::receiveDisconnect()
-{
-    qDebug() << runId << logtime() << "Client disconnected.";
-    report.end();
-    if (report.reportProduced() && !clientInfo.isAdHocRun())
-        issueMismatchNotification();
-    if (settings && settings->value("ProcessXmlResults").toBool() && !clientInfo.isAdHocRun()) {
-        // ### TBD: actually execute the processing command. For now, just generate the xml files.
-        QString xmlDir = report.writeResultsXmlFiles();
-    }
-    QThread::currentThread()->exit(0);
-}
-
-
-PlatformInfo BaselineHandler::mapPlatformInfo(const PlatformInfo& orig) const
-{
-    PlatformInfo mapped;
-    foreach (const QString &key, orig.uniqueKeys()) {
-        QString val = orig.value(key).simplified();
-        val.replace(QLC('/'), QLC('_'));
-        val.replace(QLC(' '), QLC('_'));
-        mapped.insert(key, QUrl::toPercentEncoding(val, "+"));
-        //qDebug() << "MAPPED" << key << "FROM" << orig.value(key) << "TO" << mapped.value(key);
-    }
-
-    // Special fixup for OS version
-    if (mapped.value(PI_OSName) == QLS("MacOS")) {
-        int ver = mapped.value(PI_OSVersion).toInt();
-        if (ver > 1)
-            mapped.insert(PI_OSVersion, QString("MV_10_%1").arg(ver-2));
-    }
-    else if (mapped.value(PI_OSName) == QLS("Windows")) {
-        // TBD: map windows version numbers to names
-    }
-
-    // Special fixup for hostname
-    QString host = mapped.value(PI_HostName).section(QLC('.'), 0, 0);  // Filter away domain, if any
-    if (host.isEmpty() || host == QLS("localhost")) {
-        host = orig.value(PI_HostAddress);
-    } else {
-        if (!orig.isAdHocRun()) {    // i.e. CI system run, so remove index postfix typical of vm hostnames
-            host.remove(QRegularExpression(QLS("\\d+$")));
-            if (host.endsWith(QLC('-')))
-                host.chop(1);
-        }
-    }
-    if (host.isEmpty())
-        host = QLS("UNKNOWN-HOST");
-    if (mapped.value(PI_OSName) == QLS("MacOS"))        // handle multiple os versions on same host
-        host += QLC('-') + mapped.value(PI_OSVersion);
-    mapped.insert(PI_HostName, host);
-
-    // Special fixup for Qt version
-    QString ver = mapped.value(PI_QtVersion);
-    if (!ver.isEmpty())
-        mapped.insert(PI_QtVersion, ver.prepend(QLS("Qt-")));
-
-    return mapped;
-}
-
-
-QString BaselineHandler::pathForItem(const ImageItem &item, bool isBaseline, bool absolute) const
-{
-    if (mappedClientInfo.isEmpty()) {
-        mappedClientInfo = mapPlatformInfo(clientInfo);
-        PlatformInfo oraw = clientInfo;
-        // ### simplify: don't map if no overrides!
-        for (int i = 0; i < clientInfo.overrides().size()-1; i+=2)
-            oraw.insert(clientInfo.overrides().at(i), clientInfo.overrides().at(i+1));
-        overriddenMappedClientInfo = mapPlatformInfo(oraw);
-    }
-
-    const PlatformInfo& mapped = isBaseline ? overriddenMappedClientInfo : mappedClientInfo;
-
-    QString itemName = safeName(item.itemName);
-    itemName.append(QLC('_') + QString::number(item.itemChecksum, 16).rightJustified(4, QLC('0')));
-
-    QStringList path;
-    path += projectPath(absolute);
-    path += mapped.value(PI_TestCase);
-    path += QLS(isBaseline ? "baselines" : "mismatches");
-    path += item.testFunction;
-    QStringList itemPathKeys;
-    if (settings)
-        itemPathKeys = settings->value("ItemPathKeys").toStringList();
-    if (itemPathKeys.isEmpty())
-        itemPathKeys = BaselineServer::defaultPathKeys();
-    foreach (const QString &key, itemPathKeys)
-        path += mapped.value(key, QLS("UNSET-")+key);
-    if (!isBaseline)
-        path += runId;
-    path += itemName + QLC('.');
-
-    return path.join(QLS("/"));
-}
-
-
-QString BaselineHandler::view(const QString &baseline, const QString &rendered, const QString &compared)
-{
-    QFile f(":/templates/view.html");
-    f.open(QIODevice::ReadOnly);
-    return QString::fromLatin1(f.readAll()).arg('/'+baseline, '/'+rendered, '/'+compared, diffstats(baseline, rendered));
-}
-
-QString BaselineHandler::diffstats(const QString &baseline, const QString &rendered)
-{
-    QImage blImg(BaselineServer::storagePath() + QLC('/') + baseline);
-    QImage mmImg(BaselineServer::storagePath() + QLC('/') + rendered);
-    if (blImg.isNull() || mmImg.isNull())
-        return QLS("Could not compute diffstats: image loading failed.");
-
-    // ### TBD: cache the results
-    return computeMismatchScore(blImg, mmImg);
-}
-
-QString BaselineHandler::clearAllBaselines(const QString &context)
-{
-    int tot = 0;
-    int failed = 0;
-    QDirIterator it(BaselineServer::storagePath() + QLC('/') + context,
-                    QStringList() << QLS("*.") + QLS(FileFormat)
-                                  << QLS("*.") + QLS(MetadataFileExt)
-                                  << QLS("*.") + QLS(ThumbnailExt));
-    while (it.hasNext()) {
-        bool counting = !it.next().endsWith(QLS(ThumbnailExt));
-        if (counting)
-            tot++;
-        if (!QFile::remove(it.filePath()) && counting)
-            failed++;
-    }
-    return QString(QLS("%1 of %2 baselines cleared from context ")).arg((tot-failed)/2).arg(tot/2) + context;
-}
-
-QString BaselineHandler::updateBaselines(const QString &context, const QString &mismatchContext, const QString &itemFile)
-{
-    int tot = 0;
-    int failed = 0;
-    QString storagePrefix = BaselineServer::storagePath() + QLC('/');
-    // If itemId is set, update just that one, otherwise, update all:
-    QString filter = itemFile.isEmpty() ? QLS("*_????.") : itemFile;
-    QDirIterator it(storagePrefix + mismatchContext,
-                    QStringList() << filter + QLS(FileFormat)
-                                  << filter + QLS(MetadataFileExt)
-                                  << filter + QLS(ThumbnailExt));
-    while (it.hasNext()) {
-        bool counting = !it.next().endsWith(QLS(ThumbnailExt));
-        if (counting)
-            tot++;
-        QString oldFile = storagePrefix + context + QLC('/') + it.fileName();
-        QFile::remove(oldFile);                                   // Remove existing baseline file
-        if (!QFile::copy(it.filePath(), oldFile) && counting)     // and replace it with the mismatch
-            failed++;
-    }
-    return QString(QLS("%1 of %2 baselines updated in context %3 from context %4")).arg((tot-failed)/2).arg(tot/2).arg(context, mismatchContext);
-}
-
-QString BaselineHandler::blacklistTest(const QString &context, const QString &itemId, bool removeFromBlacklist)
-{
-    QFile file(BaselineServer::storagePath() + QLC('/') + context + QLS("/BLACKLIST"));
-    QStringList blackList;
-    if (file.open(QIODevice::ReadWrite)) {
-        while (!file.atEnd())
-            blackList.append(file.readLine().trimmed());
-
-        if (removeFromBlacklist)
-            blackList.removeAll(itemId);
-        else if (!blackList.contains(itemId))
-            blackList.append(itemId);
-
-        file.resize(0);
-        foreach (QString id, blackList)
-            file.write(id.toLatin1() + '\n');
-        file.close();
-        return QLS(removeFromBlacklist ? "Whitelisted " : "Blacklisted ") + itemId + QLS(" in context ") + context;
-    } else {
-        return QLS("Unable to update blacklisted tests, failed to open ") + file.fileName();
-    }
-}
-
-
-void BaselineHandler::testPathMapping()
-{
-    qDebug() << "Storage prefix:" << BaselineServer::storagePath();
-
-    QStringList hosts;
-    hosts << QLS("bq-ubuntu910-x86-01")
-          << QLS("bq-ubuntu910-x86-15")
-          << QLS("osl-mac-master-5.test.qt-project.org")
-          << QLS("osl-mac-master-6.test.qt-project.org")
-          << QLS("sv-xp-vs-010")
-          << QLS("sv-xp-vs-011")
-          << QLS("sv-solaris-sparc-008")
-          << QLS("macbuilder-02.test.troll.no")
-          << QLS("bqvm1164")
-          << QLS("chimera")
-          << QLS("localhost")
-          << QLS("");
-
-    ImageItem item;
-    item.testFunction = QLS("testPathMapping");
-    item.itemName = QLS("arcs.qps");
-    item.imageChecksums << 0x0123456789abcdefULL;
-    item.itemChecksum = 0x0123;
-
-    clientInfo.insert(PI_QtVersion, QLS("5.0.0"));
-    clientInfo.insert(PI_QMakeSpec, QLS("linux-g++"));
-    clientInfo.insert(PI_PulseGitBranch, QLS("somebranch"));
-    clientInfo.setAdHocRun(false);
-    foreach(const QString& host, hosts) {
-        mappedClientInfo.clear();
-        clientInfo.insert(PI_HostName, host);
-        qDebug() << "Baseline from" << host << "->" << pathForItem(item, true);
-        qDebug() << "Mismatch from" << host << "->" << pathForItem(item, false);
-    }
-}
-
-
-QString BaselineHandler::computeMismatchScore(const QImage &baseline, const QImage &rendered)
-{
-    if (baseline.size() != rendered.size() || baseline.format() != rendered.format())
-        return QLS("[No diffstats, incomparable images.]");
-    if (baseline.depth() != 32)
-        return QLS("[Diffstats computation not implemented for format.]");
-
-    int w = baseline.width();
-    int h = baseline.height();
-
-    uint ncd = 0; // number of differing color pixels
-    uint nad = 0; // number of differing alpha pixels
-    uint scd = 0; // sum of color pixel difference
-    uint sad = 0; // sum of alpha pixel difference
-    uint mind = 0; // minimum difference
-    uint maxd = 0; // maximum difference
-
-    for (int y=0; y<h; ++y) {
-        const QRgb *bl = (const QRgb *) baseline.constScanLine(y);
-        const QRgb *rl = (const QRgb *) rendered.constScanLine(y);
-        for (int x=0; x<w; ++x) {
-            QRgb b = bl[x];
-            QRgb r = rl[x];
-            if (r != b) {
-                uint dr = qAbs(qRed(b) - qRed(r));
-                uint dg = qAbs(qGreen(b) - qGreen(r));
-                uint db = qAbs(qBlue(b) - qBlue(r));
-                uint ds = (dr + dg + db) / 3;
-                uint da = qAbs(qAlpha(b) - qAlpha(r));
-                if (ds) {
-                    ncd++;
-                    scd += ds;
-                    if (!mind || ds < mind)
-                        mind = ds;
-                    if (ds > maxd)
-                        maxd = ds;
-                }
-                if (da) {
-                    nad++;
-                    sad += da;
-                }
-            }
-        }
-    }
-
-
-    double pcd = 100.0 * ncd / (w*h);  // percent of pixels that differ
-    double acd = ncd ? double(scd) / (ncd) : 0;         // avg. difference
-/*
-    if (baseline.hasAlphaChannel()) {
-        double pad = 100.0 * nad / (w*h);  // percent of pixels that differ
-        double aad = nad ? double(sad) / (3*nad) : 0;         // avg. difference
-    }
-*/
-    QString res = "<table>\n";
-    QString item = "<tr><td>%1</td><td align=right>%2</td></tr>\n";
-    res += item.arg("Number of mismatching pixels").arg(ncd);
-    res += item.arg("Percentage mismatching pixels").arg(pcd, 0, 'g', 2);
-    res += item.arg("Minimum pixel distance").arg(mind);
-    res += item.arg("Maximum pixel distance").arg(maxd);
-    if (acd >= 10.0)
-        res += item.arg("Average pixel distance").arg(qRound(acd));
-    else
-        res += item.arg("Average pixel distance").arg(acd, 0, 'g', 2);
-
-    if (baseline.hasAlphaChannel())
-        res += item.arg("Number of mismatching alpha values").arg(nad);
-
-    res += "</table>\n";
-    res += "<p>(Distances are normalized to the range 0-255)</p>\n";
-    return res;
-}
-
-
-bool BaselineHandler::fuzzyCompare(const QString &baselinePath, const QString &mismatchPath)
-{
-    QProcess compareProc;
-    QStringList args;
-    args << "-fuzz" << QString("%1%").arg(fuzzLevel) << "-metric" << "AE";
-    args << baselinePath + QLS(FileFormat) << mismatchPath + QLS(FileFormat) << "/dev/null";  // TBD: Should save output image, so report won't have to regenerate it
-
-    compareProc.setProcessChannelMode(QProcess::MergedChannels);
-    compareProc.start("compare", args, QIODevice::ReadOnly);
-    if (compareProc.waitForFinished(3000) && compareProc.error() == QProcess::UnknownError) {
-        bool ok = false;
-        int metric = compareProc.readAll().trimmed().toInt(&ok);
-        if (ok && metric == 0)
-            return true;
-    }
-    return false;
-}
-
-
-void BaselineHandler::issueMismatchNotification()
-{
-    // KISS: hardcoded use of the "sendemail" utility. Make this configurable if and when demand arises.
-    if (!settings)
-        return;
-
-    settings->beginGroup("Notification");
-    QStringList receivers = settings->value("Receivers").toStringList();
-    QString sender = settings->value("Sender").toString();
-    QString server = settings->value("SMTPserver").toString();
-    settings->endGroup();
-    if (receivers.isEmpty() || sender.isEmpty() || server.isEmpty())
-        return;
-
-    QString msg = QString("\nResult summary for test run %1:\n").arg(runId);
-    msg += report.summary();
-    msg += "\nReport: " + BaselineServer::baseUrl() + report.filePath() + "\n";
-
-    msg += "\nTest run platform properties:\n------------------\n";
-    foreach (const QString &key, clientInfo.keys())
-        msg += key + ":  " + clientInfo.value(key) + '\n';
-    msg += "\nCheers,\n- Your friendly Lancelot Baseline Server\n";
-
-    QProcess proc;
-    QString cmd = "sendemail";
-    QStringList args;
-    args << "-s" << server << "-f" << sender << "-t" << receivers;
-    args << "-u" << "[Lancelot] Mismatch report for project " + clientInfo.value(PI_Project) + ", test case " + clientInfo.value(PI_TestCase);
-    args << "-m" << msg;
-
-    //proc.setProcessChannelMode(QProcess::MergedChannels);
-    proc.start(cmd, args);
-    if (!proc.waitForFinished(10 * 1000) || (proc.exitStatus() != QProcess::NormalExit) || proc.exitCode()) {
-        qWarning() << "FAILED to issue notification. Command:" << cmd << args.mid(0, args.size()-2);
-        qWarning() << "    Command standard output:" << proc.readAllStandardOutput();
-        qWarning() << "    Command error output:" << proc.readAllStandardError();
-    }
-}
-
-
-// Make an identifer safer for use as filename and URL
-QString safeName(const QString& name)
-{
-    QString res = name.simplified();
-    res.replace(QLC(' '), QLC('_'));
-    res.replace(QLC('.'), QLC('_'));
-    res.replace(QLC('/'), QLC('^'));
-    return res;
-}
author	Eirik Aavitsland <eirik.aavitsland@qt.io>	2021-11-15 13:36:48 +0100
committer	Volker Hilsheimer <volker.hilsheimer@qt.io>	2021-11-16 14:01:50 +0100
commit	e8f93e38debf4f46dbda4de198b9dd475df42d5c (patch)
tree	c042e1fea11d2b63788f1b8855030ecd96e13c19 /tests/baselineserver/src/baselineserver.cpp
parent	9a131b59ee561e300fb6078886a85dd7b3c53599 (diff)