summaryrefslogtreecommitdiffstats
path: root/tests/baselineserver/src/report.cpp
diff options
context:
space:
mode:
authorEirik Aavitsland <eirik.aavitsland@qt.io>2021-11-15 13:36:48 +0100
committerVolker Hilsheimer <volker.hilsheimer@qt.io>2021-11-16 14:01:50 +0100
commite8f93e38debf4f46dbda4de198b9dd475df42d5c (patch)
treec042e1fea11d2b63788f1b8855030ecd96e13c19 /tests/baselineserver/src/report.cpp
parent9a131b59ee561e300fb6078886a85dd7b3c53599 (diff)
Rename and restructure the baseline (lancelot) testing code
In preparation for addition of new baseline tests, establish a new test category, "baseline". This is similar to the category "benchmarks" in that it contains tests that use the QTest framework, but conceptually are not unit tests, in contrast to those under auto/. Move the existing QPainter baseline test, tst_lancelot, into this new category, and rename it accordingly. Baseline tests use the QBaselineTest extension to QTest. Move that extension too into the tests/baseline directory, allowing the clean out of the baselineserver directory. Pick-to: 6.2 Change-Id: I1b527f5867c953b1d22be73798fcf7d1494712ea Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org> Reviewed-by: Eirik Aavitsland <eirik.aavitsland@qt.io> Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>
Diffstat (limited to 'tests/baselineserver/src/report.cpp')
-rw-r--r--tests/baselineserver/src/report.cpp503
1 files changed, 0 insertions, 503 deletions
diff --git a/tests/baselineserver/src/report.cpp b/tests/baselineserver/src/report.cpp
deleted file mode 100644
index 748d76ebfef..00000000000
--- a/tests/baselineserver/src/report.cpp
+++ /dev/null
@@ -1,503 +0,0 @@
-/****************************************************************************
-**
-** Copyright (C) 2016 The Qt Company Ltd.
-** Contact: https://www.qt.io/licensing/
-**
-** This file is part of the test suite of the Qt Toolkit.
-**
-** $QT_BEGIN_LICENSE:GPL-EXCEPT$
-** Commercial License Usage
-** Licensees holding valid commercial Qt licenses may use this file in
-** accordance with the commercial license agreement provided with the
-** Software or, alternatively, in accordance with the terms contained in
-** a written agreement between you and The Qt Company. For licensing terms
-** and conditions see https://www.qt.io/terms-conditions. For further
-** information use the contact form at https://www.qt.io/contact-us.
-**
-** GNU General Public License Usage
-** Alternatively, this file may be used under the terms of the GNU
-** General Public License version 3 as published by the Free Software
-** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
-** included in the packaging of this file. Please review the following
-** information to ensure the GNU General Public License requirements will
-** be met: https://www.gnu.org/licenses/gpl-3.0.html.
-**
-** $QT_END_LICENSE$
-**
-****************************************************************************/
-#include "report.h"
-#include "baselineprotocol.h"
-#include "baselineserver.h"
-#include <QDir>
-#include <QProcess>
-#include <QUrl>
-#include <QXmlStreamWriter>
-#include <QRegularExpression>
-#include <unistd.h>
-
-Report::Report()
- : initialized(false), handler(0), written(false), numItems(0), numMismatches(0), settings(0),
- hasStats(false)
-{
-}
-
-Report::~Report()
-{
- end();
-}
-
-QString Report::filePath()
-{
- return path;
-}
-
-int Report::numberOfMismatches()
-{
- return numMismatches;
-}
-
-bool Report::reportProduced()
-{
- return written;
-}
-
-void Report::init(const BaselineHandler *h, const QString &r, const PlatformInfo &p, const QSettings *s)
-{
- handler = h;
- runId = r;
- plat = p;
- settings = s;
- rootDir = BaselineServer::storagePath() + QLC('/');
- baseDir = handler->pathForItem(ImageItem(), true, false).remove(QRegularExpression("/baselines/.*$"));
- QString dir = baseDir + (plat.isAdHocRun() ? QLS("/adhoc-reports") : QLS("/auto-reports"));
- QDir cwd;
- if (!cwd.exists(rootDir + dir))
- cwd.mkpath(rootDir + dir);
- path = dir + QLS("/Report_") + runId + QLS(".html");
- hasOverride = !plat.overrides().isEmpty();
- initialized = true;
-}
-
-void Report::addItems(const ImageItemList &items)
-{
- if (items.isEmpty())
- return;
- numItems += items.size();
- QString func = items.at(0).testFunction;
- if (!testFunctions.contains(func))
- testFunctions.append(func);
- ImageItemList list = items;
- if (settings->value("ReportMissingResults").toBool()) {
- for (ImageItemList::iterator it = list.begin(); it != list.end(); ++it) {
- if (it->status == ImageItem::Ok)
- it->status = ImageItem::Error; // Status should be set by report from client, else report as error
- }
- }
- itemLists[func] += list;
-}
-
-void Report::addResult(const ImageItem &item)
-{
- if (!testFunctions.contains(item.testFunction)) {
- qWarning() << "Report::addResult: unknown testfunction" << item.testFunction;
- return;
- }
- bool found = false;
- ImageItemList &list = itemLists[item.testFunction];
- for (ImageItemList::iterator it = list.begin(); it != list.end(); ++it) {
- if (it->itemName == item.itemName && it->itemChecksum == item.itemChecksum) {
- it->status = item.status;
- found = true;
- break;
- }
- }
- if (found) {
- if (item.status == ImageItem::Mismatch)
- numMismatches++;
- } else {
- qWarning() << "Report::addResult: unknown item" << item.itemName << "in testfunction" << item.testFunction;
- }
-}
-
-void Report::end()
-{
- if (!initialized || written)
- return;
- // Make report iff (#mismatches>0) || (#fuzzymatches>0) || (#errors>0 && settings say report errors)
- bool doReport = (numMismatches > 0);
- if (!doReport) {
- bool reportErrors = settings->value("ReportMissingResults").toBool();
- computeStats();
- foreach (const QString &func, itemLists.keys()) {
- FuncStats stat = stats.value(func);
- if (stat.value(ImageItem::FuzzyMatch) > 0) {
- doReport = true;
- break;
- }
- foreach (const ImageItem &item, itemLists.value(func)) {
- if (reportErrors && item.status == ImageItem::Error) {
- doReport = true;
- break;
- }
- }
- if (doReport)
- break;
- }
- }
- if (!doReport)
- return;
- write();
- written = true;
-}
-
-void Report::computeStats()
-{
- if (hasStats)
- return;
- foreach (const QString &func, itemLists.keys()) {
- FuncStats funcStat;
- funcStat[ImageItem::Ok] = 0;
- funcStat[ImageItem::BaselineNotFound] = 0;
- funcStat[ImageItem::IgnoreItem] = 0;
- funcStat[ImageItem::Mismatch] = 0;
- funcStat[ImageItem::FuzzyMatch] = 0;
- funcStat[ImageItem::Error] = 0;
- foreach (const ImageItem &item, itemLists.value(func)) {
- funcStat[item.status]++;
- }
- stats[func] = funcStat;
- }
- hasStats = true;
-}
-
-QString Report::summary()
-{
- computeStats();
- QString res;
- foreach (const QString &func, itemLists.keys()) {
- FuncStats stat = stats.value(func);
- QString s = QString("%1 %3 mismatch(es), %4 error(s), %5 fuzzy match(es)\n");
- s = s.arg(QString("%1() [%2 items]:").arg(func).arg(itemLists.value(func).size()).leftJustified(40));
- s = s.arg(stat.value(ImageItem::Mismatch));
- s = s.arg(stat.value(ImageItem::Error));
- s = s.arg(stat.value(ImageItem::FuzzyMatch));
- res += s;
- }
-#if 0
- qDebug() << "***************************** Summary *************************";
- qDebug() << res;
- qDebug() << "***************************************************************";
-#endif
- return res;
-}
-
-void Report::write()
-{
- QFile file(rootDir + path);
- if (!file.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
- qWarning() << "Failed to open report file" << file.fileName();
- return;
- }
- out.setDevice(&file);
-
- writeHeader();
- foreach(const QString &func, testFunctions) {
- writeFunctionResults(itemLists.value(func));
- }
- writeFooter();
- file.close();
- updateLatestPointer();
-}
-
-
-void Report::writeHeader()
-{
- QString title = plat.value(PI_Project) + QLC(':') + plat.value(PI_TestCase) + QLS(" Lancelot Test Report");
- out << "<!DOCTYPE html>\n"
- << "<html><head><title>" << title << "</title></head>\n"
- << "<body bgcolor=""#ddeeff""><h1>" << title << "</h1>\n"
- << "<p>Note: This is a <i>static</i> page, generated at " << QDateTime::currentDateTime().toString()
- << " for the test run with id " << runId << "</p>\n"
- << "<p>Summary: <b><span style=\"color:red\">" << numMismatches << " of " << numItems << "</span></b> items reported mismatching</p>\n";
- out << "<pre>\n" << summary() << "</pre>\n\n";
- out << "<h3>Testing Client Platform Info:</h3>\n"
- << "<table>\n";
- foreach (QString key, plat.keys())
- out << "<tr><td>" << key << ":</td><td>" << plat.value(key) << "</td></tr>\n";
- out << "</table>\n\n";
- if (hasOverride) {
- out << "<span style=\"color:red\"><h4>Note! Override Platform Info:</h4></span>\n"
- << "<p>The client's output has been compared to baselines created on a different platform. Differences:</p>\n"
- << "<table>\n";
- for (int i = 0; i < plat.overrides().size()-1; i+=2)
- out << "<tr><td>" << plat.overrides().at(i) << ":</td><td>" << plat.overrides().at(i+1) << "</td></tr>\n";
- out << "</table>\n\n";
- }
-}
-
-
-void Report::writeFunctionResults(const ImageItemList &list)
-{
- QString testFunction = list.at(0).testFunction;
- QString pageUrl = BaselineServer::baseUrl() + path;
- QString ctx = handler->pathForItem(list.at(0), true, false).section(QLC('/'), 0, -2);
- QString misCtx = handler->pathForItem(list.at(0), false, false).section(QLC('/'), 0, -2);
-
-
- out << "\n<p>&nbsp;</p><h3>Test function: " << testFunction << "</h3>\n";
- if (!hasOverride) {
- out << "<p><a href=\"/cgi-bin/server.cgi?cmd=clearAllBaselines&context=" << ctx << "&url=" << pageUrl
- << "\"><b>Clear all baselines</b></a> for this testfunction (They will be recreated by the next run)</p>\n";
- out << "<p><a href=\"/cgi-bin/server.cgi?cmd=updateAllBaselines&context=" << ctx << "&mismatchContext=" << misCtx << "&url=" << pageUrl
- << "\"><b>Let these mismatching images be the new baselines</b></a> for this testfunction</p>\n\n";
- }
-
- out << "<table border=\"2\">\n"
- "<tr>\n"
- "<th width=123>Item</th>\n"
- "<th width=246>Baseline</th>\n"
- "<th width=246>Rendered</th>\n"
- "<th width=246>Comparison (diffs are <span style=\"color:red\">RED</span>)</th>\n"
- "<th width=246>Info/Action</th>\n"
- "</tr>\n\n";
-
- foreach (const ImageItem &item, list) {
- QString mmPrefix = handler->pathForItem(item, false, false);
- QString blPrefix = handler->pathForItem(item, true, false);
-
- // Make hard links to the current baseline, so that the report is static even if the baseline changes
- generateThumbnail(blPrefix + QLS(FileFormat), rootDir); // Make sure baseline thumbnail is up to date
- QString lnPrefix = mmPrefix + QLS("baseline.");
- QByteArray blPrefixBa = (rootDir + blPrefix).toLatin1();
- QByteArray lnPrefixBa = (rootDir + lnPrefix).toLatin1();
- ::link((blPrefixBa + FileFormat).constData(), (lnPrefixBa + FileFormat).constData());
- ::link((blPrefixBa + MetadataFileExt).constData(), (lnPrefixBa + MetadataFileExt).constData());
- ::link((blPrefixBa + ThumbnailExt).constData(), (lnPrefixBa + ThumbnailExt).constData());
-
- QString baseline = lnPrefix + QLS(FileFormat);
- QString metadata = lnPrefix + QLS(MetadataFileExt);
- out << "<tr>\n";
- out << "<td>" << item.itemName << "</td>\n";
- if (item.status == ImageItem::Mismatch || item.status == ImageItem::FuzzyMatch) {
- QString rendered = mmPrefix + QLS(FileFormat);
- QString itemFile = mmPrefix.section(QLC('/'), -1);
- writeItem(baseline, rendered, item, itemFile, ctx, misCtx, metadata);
- }
- else {
- out << "<td align=center><a href=\"/" << baseline << "\">image</a> <a href=\"/" << metadata << "\">info</a></td>\n"
- << "<td align=center colspan=2><small>n/a</small></td>\n"
- << "<td align=center>";
- switch (item.status) {
- case ImageItem::BaselineNotFound:
- out << "Baseline not found/regenerated";
- break;
- case ImageItem::IgnoreItem:
- out << "<span style=\"background-color:yellow\">Blacklisted</span> ";
- if (!hasOverride) {
- out << "<a href=\"/cgi-bin/server.cgi?cmd=whitelist&context=" << ctx
- << "&itemId=" << item.itemName << "&url=" << pageUrl
- << "\">Whitelist this item</a>";
- }
- break;
- case ImageItem::Error:
- out << "<span style=\"background-color:red\">Error: No result reported!</span>";
- break;
- case ImageItem::Ok:
- out << "<span style=\"color:green\"><small>No mismatch reported</small></span>";
- break;
- default:
- out << '?';
- break;
- }
- out << "</td>\n";
- }
- out << "</tr>\n\n";
- }
-
- out << "</table>\n";
-}
-
-void Report::writeItem(const QString &baseline, const QString &rendered, const ImageItem &item,
- const QString &itemFile, const QString &ctx, const QString &misCtx, const QString &metadata)
-{
- QString compared = generateCompared(baseline, rendered);
- QString pageUrl = BaselineServer::baseUrl() + path;
-
- QStringList images = QStringList() << baseline << rendered << compared;
- foreach (const QString& img, images)
- out << "<td height=246 align=center><a href=\"/" << img << "\"><img src=\"/" << generateThumbnail(img, rootDir) << "\"></a></td>\n";
-
- out << "<td align=center>\n";
- if (item.status == ImageItem::FuzzyMatch)
- out << "<p><span style=\"color:orange\">Fuzzy match</span></p>\n";
- else
- out << "<p><span style=\"color:red\">Mismatch reported</span></p>\n";
- out << "<p><a href=\"/" << metadata << "\">Baseline Info</a>\n";
- if (!hasOverride) {
- out << "<p><a href=\"/cgi-bin/server.cgi?cmd=updateSingleBaseline&context=" << ctx << "&mismatchContext=" << misCtx
- << "&itemFile=" << itemFile << "&url=" << pageUrl << "\">Let this be the new baseline</a></p>\n"
- << "<p><a href=\"/cgi-bin/server.cgi?cmd=blacklist&context=" << ctx
- << "&itemId=" << item.itemName << "&url=" << pageUrl << "\">Blacklist this item</a></p>\n";
- }
- out << "<p><a href=\"/cgi-bin/server.cgi?cmd=view&baseline=" << baseline << "&rendered=" << rendered
- << "&compared=" << compared << "&url=" << pageUrl << "\">Inspect</a></p>\n";
-
-#if 0
- out << "<p><a href=\"/cgi-bin/server.cgi?cmd=diffstats&baseline=" << baseline << "&rendered=" << rendered
- << "&url=" << pageUrl << "\">Diffstats</a></p>\n";
-#endif
-
- out << "</td>\n";
-}
-
-void Report::writeFooter()
-{
- out << "\n</body></html>\n";
-}
-
-
-QString Report::generateCompared(const QString &baseline, const QString &rendered, bool fuzzy)
-{
- QString res = rendered;
- QFileInfo fi(res);
- res.chop(fi.suffix().length());
- res += QLS(fuzzy ? "fuzzycompared.png" : "compared.png");
- QStringList args;
- if (fuzzy)
- args << QLS("-fuzz") << QLS("5%");
- args << rootDir+baseline << rootDir+rendered << rootDir+res;
- QProcess::execute(QLS("compare"), args);
- return res;
-}
-
-
-QString Report::generateThumbnail(const QString &image, const QString &rootDir)
-{
- QString res = image;
- QFileInfo imgFI(rootDir+image);
- if (!imgFI.exists())
- return res;
- res.chop(imgFI.suffix().length());
- res += ThumbnailExt;
- QFileInfo resFI(rootDir+res);
- if (resFI.exists() && resFI.lastModified() > imgFI.lastModified())
- return res;
- QStringList args;
- args << rootDir+image << QLS("-resize") << QLS("240x240>") << QLS("-quality") << QLS("50") << rootDir+res;
- QProcess::execute(QLS("convert"), args);
- return res;
-}
-
-
-QString Report::writeResultsXmlFiles()
-{
- if (!itemLists.size())
- return QString();
- QString dir = rootDir + baseDir + QLS("/xml-reports/") + runId;
- QDir cwd;
- if (!cwd.exists(dir))
- cwd.mkpath(dir);
- foreach (const QString &func, itemLists.keys()) {
- QFile f(dir + QLatin1Char('/') + func + "-results.xml");
- if (!f.open(QIODevice::WriteOnly))
- continue;
- QXmlStreamWriter s(&f);
- s.setAutoFormatting(true);
- s.writeStartDocument();
- foreach (QString key, plat.keys()) {
- QString cmt = QLatin1Char(' ') + key + "=\"" + plat.value(key) +"\" ";
- s.writeComment(cmt.replace("--", "[-]"));
- }
- s.writeStartElement("testsuite");
- s.writeAttribute("name", func);
- foreach (const ImageItem &item, itemLists.value(func)) {
- QString res;
- switch (item.status) {
- case ImageItem::Ok:
- case ImageItem::FuzzyMatch:
- res = "pass";
- break;
- case ImageItem::Mismatch:
- case ImageItem::Error:
- res = "fail";
- break;
- case ImageItem::BaselineNotFound:
- case ImageItem::IgnoreItem:
- default:
- res = "skip";
- }
- s.writeStartElement("testcase");
- s.writeAttribute("name", item.itemName);
- s.writeAttribute("result", res);
- s.writeEndElement();
- }
- s.writeEndElement();
- s.writeEndDocument();
- }
- return dir;
-}
-
-
-
-void Report::updateLatestPointer()
-{
- QString linkPath = rootDir + baseDir + QLS("/latest_report.html");
- QString reportPath = path.mid(baseDir.size()+1);
- QFile::remove(linkPath); // possible race with another thread, yada yada yada
- QFile::link(reportPath, linkPath);
-
-#if 0
- QByteArray fwd = "<!DOCTYPE html><html><head><meta HTTP-EQUIV=\"refresh\" CONTENT=\"0;URL=%1\"></meta></head><body></body></html>\n";
- fwd.replace("%1", filePath().prepend(QLC('/')).toLatin1());
-
- QFile file(rootDir + baseDir + "/latest_report.html");
- if (file.open(QIODevice::WriteOnly | QIODevice::Truncate))
- file.write(fwd);
-#endif
-}
-
-
-void Report::handleCGIQuery(const QString &query)
-{
- QUrl cgiUrl(QLS("http://dummy/cgi-bin/dummy.cgi?") + query);
- QTextStream s(stdout);
- s << "Content-Type: text/html\r\n\r\n"
- << "<!DOCTYPE html>\n<HTML>\n<body bgcolor=""#ddeeff"">\n"; // Lancelot blue
-
- QString command(cgiUrl.queryItemValue("cmd"));
-
- if (command == QLS("view")) {
- s << BaselineHandler::view(cgiUrl.queryItemValue(QLS("baseline")),
- cgiUrl.queryItemValue(QLS("rendered")),
- cgiUrl.queryItemValue(QLS("compared")));
- }
-#if 0
- else if (command == QLS("diffstats")) {
- s << BaselineHandler::diffstats(cgiUrl.queryItemValue(QLS("baseline")),
- cgiUrl.queryItemValue(QLS("rendered")));
- }
-#endif
- else if (command == QLS("updateSingleBaseline")) {
- s << BaselineHandler::updateBaselines(cgiUrl.queryItemValue(QLS("context")),
- cgiUrl.queryItemValue(QLS("mismatchContext")),
- cgiUrl.queryItemValue(QLS("itemFile")));
- } else if (command == QLS("updateAllBaselines")) {
- s << BaselineHandler::updateBaselines(cgiUrl.queryItemValue(QLS("context")),
- cgiUrl.queryItemValue(QLS("mismatchContext")),
- QString());
- } else if (command == QLS("clearAllBaselines")) {
- s << BaselineHandler::clearAllBaselines(cgiUrl.queryItemValue(QLS("context")));
- } else if (command == QLS("blacklist")) {
- // blacklist a test
- s << BaselineHandler::blacklistTest(cgiUrl.queryItemValue(QLS("context")),
- cgiUrl.queryItemValue(QLS("itemId")));
- } else if (command == QLS("whitelist")) {
- // whitelist a test
- s << BaselineHandler::blacklistTest(cgiUrl.queryItemValue(QLS("context")),
- cgiUrl.queryItemValue(QLS("itemId")), true);
- } else {
- s << "Unknown query:<br>" << query << "<br>";
- }
- s << "<p><a href=\"" << cgiUrl.queryItemValue(QLS("url")) << "\">Back to report</a>\n";
- s << "</body>\n</HTML>";
-}
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-17 18:49:20 +0000