nmWTAI-Platform/Src/nmNum/nmData/nmNumericalSaveSession.cpp

#include "nmNumericalSaveSession.h"

#include "nmDataAnalyzeManager.h"
#include "nmDataJsonTools.h"

#include <QDir>
#include <QFile>
#include <QFileInfo>
#include <QSet>
#include <QUuid>

#include <rapidjson/document.h>

#if defined(Q_OS_WIN)
#include <windows.h>
#include <io.h>
#endif

namespace {

const char* g_pManifestFileName = "NumericalSaveManifest.json";
const quint64 g_nMaximumManifestBytes = 16ull * 1024ull * 1024ull;
const int g_nMaximumWindowCount = 10000;

bool setError(QString* pError, const QString& sError)
{
    if(pError != NULL)
    {
        *pError = sError;
    }
    return false;
}

rapidjson::Value toJsonString(const QString& sValue,
                              rapidjson::Document::AllocatorType& oAllocator)
{
    const QByteArray baValue = sValue.toUtf8();
    return rapidjson::Value(baValue.constData(),
                            static_cast<rapidjson::SizeType>(baValue.size()),
                            oAllocator);
}

void addFileReference(rapidjson::Value& oParent,
                      const char* pName,
                      const nmNumericalFileReference& oReference,
                      rapidjson::Document::AllocatorType& oAllocator)
{
    rapidjson::Value oJson(rapidjson::kObjectType);
    oJson.AddMember("Path", toJsonString(oReference.m_sRelativePath,
                                          oAllocator), oAllocator);
    oJson.AddMember("Length", oReference.m_nLength, oAllocator);
    oJson.AddMember("Sha1", toJsonString(oReference.m_sSha1,
                                          oAllocator), oAllocator);
    oParent.AddMember(rapidjson::Value(pName, oAllocator).Move(),
                      oJson, oAllocator);
}

bool readFileReference(const rapidjson::Value& oParent,
                       const char* pName,
                       nmNumericalFileReference& oReference)
{
    if(!oParent.IsObject() || !oParent.HasMember(pName) ||
       !oParent[pName].IsObject())
    {
        return false;
    }
    const rapidjson::Value& oJson = oParent[pName];
    if(!oJson.HasMember("Path") || !oJson["Path"].IsString() ||
       !oJson.HasMember("Length") || !oJson["Length"].IsUint64() ||
       !oJson.HasMember("Sha1") || !oJson["Sha1"].IsString())
    {
        return false;
    }
    oReference.m_sRelativePath = QString::fromUtf8(oJson["Path"].GetString());
    oReference.m_nLength = oJson["Length"].GetUint64();
    oReference.m_sSha1 = QString::fromLatin1(oJson["Sha1"].GetString());
    return oReference.isValid();
}

bool removeDirectoryTree(const QString& sDirectory)
{
    const QFileInfo oInfo(sDirectory);
    if(!oInfo.exists())
    {
        return true;
    }
    if(!oInfo.isDir())
    {
        return false;
    }
    QDir oDirectory(sDirectory);
    const QFileInfoList listEntries = oDirectory.entryInfoList(
            QDir::NoDotAndDotDot | QDir::AllEntries | QDir::Hidden |
            QDir::System, QDir::DirsFirst);
    for(int nIndex = 0; nIndex < listEntries.size(); ++nIndex)
    {
        const QFileInfo& oEntry = listEntries[nIndex];
        if(oEntry.isDir())
        {
            if(!removeDirectoryTree(oEntry.absoluteFilePath()))
            {
                return false;
            }
        }
        else if(!QFile::remove(oEntry.absoluteFilePath()))
        {
            return false;
        }
    }
    return QDir().rmdir(sDirectory);
}

bool loadManifest(const QString& sResultRootDirectory,
                  QMap<QString, nmNumericalSavedWindow>& mapWindows,
                  QString* pGenerationId,
                  QString* pError)
{
    mapWindows.clear();
    const QString sManifestPath = QDir(sResultRootDirectory).filePath(
            g_pManifestFileName);
    const QFileInfo oInfo(sManifestPath);
    if(!oInfo.isFile() || oInfo.size() <= 0 ||
       static_cast<quint64>(oInfo.size()) > g_nMaximumManifestBytes)
    {
        return setError(pError, "Numerical v3 root manifest is missing or too large.");
    }
    rapidjson::Document oDocument;
    if(!nmDataJsonTools::ReadDomFromFile(sManifestPath, oDocument) ||
       !oDocument.IsObject() ||
       !oDocument.HasMember("NumericalProjectVersion") ||
       !oDocument["NumericalProjectVersion"].IsInt() ||
       oDocument["NumericalProjectVersion"].GetInt() !=
            nmNumericalResultPersistence::projectVersion() ||
       !oDocument.HasMember("ManifestFormatVersion") ||
       !oDocument["ManifestFormatVersion"].IsInt() ||
       oDocument["ManifestFormatVersion"].GetInt() != 1 ||
       !oDocument.HasMember("GenerationId") ||
       !oDocument["GenerationId"].IsString() ||
       !oDocument.HasMember("Windows") ||
       !oDocument["Windows"].IsArray() ||
       oDocument["Windows"].Size() >
            static_cast<rapidjson::SizeType>(g_nMaximumWindowCount))
    {
        return setError(pError, "Numerical v3 root manifest is invalid.");
    }
    const QString sGenerationId = QString::fromUtf8(
            oDocument["GenerationId"].GetString());
    QString sNormalizedGenerationId;
    if(!nmNumericalResultPersistence::normalizeRelativePath(
                sGenerationId, sNormalizedGenerationId) ||
       sNormalizedGenerationId != sGenerationId ||
       sNormalizedGenerationId.contains('/'))
    {
        return setError(pError, "Numerical v3 root manifest generation is invalid.");
    }
    if(pGenerationId != NULL)
    {
        *pGenerationId = sGenerationId;
    }
    const rapidjson::Value& oWindows = oDocument["Windows"];
    QSet<QString> setReferencedPaths;
    bool bReferencesCurrentGeneration = false;
    for(rapidjson::SizeType nIndex = 0; nIndex < oWindows.Size(); ++nIndex)
    {
        const rapidjson::Value& oJson = oWindows[nIndex];
        if(!oJson.IsObject() ||
           !oJson.HasMember("ResultId") || !oJson["ResultId"].IsString() ||
           !oJson.HasMember("WindowId") || !oJson["WindowId"].IsString())
        {
            return setError(pError, "Numerical root manifest window is invalid.");
        }
        nmNumericalSavedWindow oWindow;
        oWindow.m_sResultId = QString::fromUtf8(oJson["ResultId"].GetString());
        oWindow.m_sWindowId = QString::fromUtf8(oJson["WindowId"].GetString());
        QString sNormalizedWindowId;
        if(oWindow.m_sResultId.isEmpty() ||
           !nmNumericalResultPersistence::normalizeRelativePath(
                oWindow.m_sWindowId, sNormalizedWindowId) ||
           sNormalizedWindowId != oWindow.m_sWindowId ||
           sNormalizedWindowId.contains('/') ||
           mapWindows.contains(oWindow.m_sWindowId) ||
           !readFileReference(oJson, "MainJson", oWindow.m_oMainJson) ||
           setReferencedPaths.contains(oWindow.m_oMainJson.m_sRelativePath))
        {
            return setError(pError, "Numerical root manifest contains duplicate or invalid windows.");
        }
        // 根清单只接受固定代次目录，窗口身份不能与物理目录脱节。
        const QStringList listPathParts =
                oWindow.m_oMainJson.m_sRelativePath.split('/');
        if(listPathParts.size() != 5 ||
           listPathParts[0] != "NumericalGenerations" ||
           listPathParts[2] != "Windows" ||
           listPathParts[3] != oWindow.m_sWindowId ||
           listPathParts[4] != "Numerical_Parameters.json")
        {
            return setError(pError, "Numerical root manifest window path is invalid.");
        }
        if(listPathParts[1] == sGenerationId)
        {
            bReferencesCurrentGeneration = true;
        }
        QString sMainPath;
        if(!nmNumericalResultPersistence::validateFileReference(
                    sResultRootDirectory, oWindow.m_oMainJson,
                    &sMainPath, pError))
        {
            return false;
        }
        setReferencedPaths.insert(oWindow.m_oMainJson.m_sRelativePath);
        mapWindows.insert(oWindow.m_sWindowId, oWindow);
    }
    // 删除全部数值分析窗口时允许发布空清单；它明确表示本成果当前没有数值载荷。
    if(mapWindows.isEmpty())
    {
        const QString sGenerationDirectory = QDir(
                sResultRootDirectory).filePath(
                    "NumericalGenerations/" + sGenerationId);
        return QFileInfo(sGenerationDirectory).isDir()
                ? true
                : setError(pError, "Empty numerical manifest generation is missing.");
    }
    return bReferencesCurrentGeneration
            ? true
            : setError(pError, "Numerical root manifest does not reference its current generation.");
}

bool flushFileToDisk(const QString& sFilePath)
{
    QFile oFile(sFilePath);
    if(!oFile.open(QIODevice::ReadWrite) || !oFile.flush())
    {
        oFile.close();
        return false;
    }
#if defined(Q_OS_WIN)
    const intptr_t nNativeHandle = _get_osfhandle(oFile.handle());
    const bool bFlushed = nNativeHandle != -1 &&
            FlushFileBuffers(reinterpret_cast<HANDLE>(nNativeHandle)) != FALSE;
#else
    const bool bFlushed = true;
#endif
    oFile.close();
    return bFlushed;
}

bool publishManifestFile(const QString& sTargetPath,
                         const QString& sTemporaryPath,
                         const QString& sBackupPath)
{
#if defined(Q_OS_WIN)
    const std::wstring wsTarget = QDir::toNativeSeparators(
            sTargetPath).toStdWString();
    const std::wstring wsTemporary = QDir::toNativeSeparators(
            sTemporaryPath).toStdWString();
    const std::wstring wsBackup = QDir::toNativeSeparators(
            sBackupPath).toStdWString();
    if(QFileInfo(sTargetPath).exists())
    {
        QFile::remove(sBackupPath);
        return ReplaceFileW(wsTarget.c_str(), wsTemporary.c_str(),
                wsBackup.c_str(), 0, NULL, NULL) != FALSE;
    }
    return MoveFileExW(wsTemporary.c_str(), wsTarget.c_str(),
                       MOVEFILE_WRITE_THROUGH) != FALSE;
#else
    Q_UNUSED(sBackupPath);
    if(QFileInfo(sTargetPath).exists() && !QFile::remove(sTargetPath))
    {
        return false;
    }
    return QFile::rename(sTemporaryPath, sTargetPath);
#endif
}

bool restoreManifestBackup(const QString& sTargetPath,
                           const QString& sBackupPath)
{
    if(!QFileInfo(sBackupPath).isFile())
    {
        return false;
    }
#if defined(Q_OS_WIN)
    const std::wstring wsTarget = QDir::toNativeSeparators(
            sTargetPath).toStdWString();
    const std::wstring wsBackup = QDir::toNativeSeparators(
            sBackupPath).toStdWString();
    return ReplaceFileW(wsTarget.c_str(), wsBackup.c_str(),
                        NULL, 0, NULL, NULL) != FALSE;
#else
    QFile::remove(sTargetPath);
    return QFile::rename(sBackupPath, sTargetPath);
#endif
}

}

nmNumericalSaveSession::nmNumericalSaveSession()
    : m_bActive(false),
      m_bCommitted(false),
      m_bKeepGenerationOnAbort(false)
{
}

nmNumericalSaveSession::~nmNumericalSaveSession()
{
    if(m_bActive && !m_bCommitted)
    {
        abort();
    }
}

bool nmNumericalSaveSession::begin(
        const QString& sResultRootDirectory,
        const QString& sResultId,
        bool bPreserveExistingWindows,
        QString* pError)
{
    if(pError != NULL)
    {
        pError->clear();
    }
    if(m_bActive)
    {
        abort();
    }
    if(sResultRootDirectory.isEmpty() || sResultId.isEmpty() ||
       !QDir().mkpath(sResultRootDirectory))
    {
        return setError(pError, "Cannot start numerical save session.");
    }

    m_sResultRootDirectory = QFileInfo(
            sResultRootDirectory).absoluteFilePath();
    m_sResultId = sResultId;
    m_sGenerationId = QUuid::createUuid().toString().remove('{').remove('}');
    m_sGenerationDirectory = QDir(m_sResultRootDirectory).filePath(
            "NumericalGenerations/" + m_sGenerationId);
    m_mapWindows.clear();
    m_setValidatedWindowIds.clear();
    m_bKeepGenerationOnAbort = false;

    if(bPreserveExistingWindows && QFileInfo(QDir(
            m_sResultRootDirectory).filePath(g_pManifestFileName)).isFile())
    {
        if(!loadManifest(m_sResultRootDirectory, m_mapWindows, NULL, pError))
        {
            m_mapWindows.clear();
            return false;
        }
    }
    if(!QDir().mkpath(QDir(m_sGenerationDirectory).filePath("Windows")))
    {
        m_mapWindows.clear();
        return setError(pError, "Cannot create numerical generation directory.");
    }
    m_bActive = true;
    m_bCommitted = false;
    return true;
}

bool nmNumericalSaveSession::saveWindowGeneration(
        const QString& sWindowId,
        nmDataAnalyzeManager* pDataManager,
        QString* pError)
{
    if(pError != NULL)
    {
        pError->clear();
    }
    QString sNormalizedWindowId;
    if(!m_bActive || pDataManager == NULL ||
       !nmNumericalResultPersistence::normalizeRelativePath(
            sWindowId, sNormalizedWindowId) ||
       sNormalizedWindowId.contains('/'))
    {
        return setError(pError, "Numerical save window identifier is invalid.");
    }

    const QString sRelativeWindowDirectory = QString(
            "NumericalGenerations/%1/Windows/%2")
            .arg(m_sGenerationId, sNormalizedWindowId);
    const QString sWindowDirectory = QDir(m_sResultRootDirectory).filePath(
            sRelativeWindowDirectory);
    if(!QDir().mkpath(sWindowDirectory))
    {
        return setError(pError, "Cannot create numerical window generation.");
    }

    nmNumericalFileReference oWindowMainReference;
    if(!pDataManager->saveNmResultV3(sWindowDirectory,
                                     oWindowMainReference, pError))
    {
        return false;
    }
    oWindowMainReference.m_sRelativePath = sRelativeWindowDirectory +
            "/Numerical_Parameters.json";
    if(!oWindowMainReference.isValid() ||
       !nmNumericalResultPersistence::validateFileReference(
            m_sResultRootDirectory, oWindowMainReference, NULL, pError))
    {
        return false;
    }

    nmNumericalSavedWindow oWindow;
    oWindow.m_sResultId = m_sResultId;
    oWindow.m_sWindowId = sNormalizedWindowId;
    oWindow.m_oMainJson = oWindowMainReference;
    m_mapWindows.insert(sNormalizedWindowId, oWindow);
    // Manager 已在返回前完整校验本窗口，提交时无需重复扫描大压力文件。
    m_setValidatedWindowIds.insert(sNormalizedWindowId);
    return true;
}

bool nmNumericalSaveSession::commit(QString* pError)
{
    QSet<QString> setReferencedGenerations;
    if(pError != NULL)
    {
        pError->clear();
    }
    if(!m_bActive)
    {
        return setError(pError, "Numerical save session is not active.");
    }

    rapidjson::Document oDocument;
    oDocument.SetObject();
    rapidjson::Document::AllocatorType& oAllocator = oDocument.GetAllocator();
    oDocument.AddMember("NumericalProjectVersion",
            nmNumericalResultPersistence::projectVersion(), oAllocator);
    oDocument.AddMember("ManifestFormatVersion", 1, oAllocator);
    oDocument.AddMember("GenerationId",
            toJsonString(m_sGenerationId, oAllocator), oAllocator);
    rapidjson::Value oWindows(rapidjson::kArrayType);
    QMap<QString, nmNumericalSavedWindow>::const_iterator oIt =
            m_mapWindows.constBegin();
    for(; oIt != m_mapWindows.constEnd(); ++oIt)
    {
        const nmNumericalSavedWindow& oWindow = oIt.value();
        QString sMainJsonPath;
        if(!nmNumericalResultPersistence::validateFileReference(
                    m_sResultRootDirectory, oWindow.m_oMainJson,
                    &sMainJsonPath, pError))
        {
            return false;
        }
        // 单窗口兼容会话会保留旧窗口；发布新清单前也必须复核其完整引用链。
        if(!m_setValidatedWindowIds.contains(oIt.key()))
        {
            const bool bWindowValid =
                    nmDataAnalyzeManager::validateNmResultV3Window(
                    QFileInfo(sMainJsonPath).absolutePath(),
                    sMainJsonPath, pError);
            if(!bWindowValid)
            {
                return false;
            }
        }
        rapidjson::Value oJson(rapidjson::kObjectType);
        oJson.AddMember("ResultId", toJsonString(
                oWindow.m_sResultId, oAllocator), oAllocator);
        oJson.AddMember("WindowId", toJsonString(
                oWindow.m_sWindowId, oAllocator), oAllocator);
        addFileReference(oJson, "MainJson", oWindow.m_oMainJson, oAllocator);
        oWindows.PushBack(oJson, oAllocator);
    }
    oDocument.AddMember("Windows", oWindows, oAllocator);

    const QString sManifestPath = QDir(m_sResultRootDirectory).filePath(
            g_pManifestFileName);
    const bool bHadPreviousManifest = QFileInfo(sManifestPath).isFile();
    const QString sTemporaryPath = sManifestPath + ".tmp." + m_sGenerationId;
    const QString sBackupPath = sManifestPath + ".bak." + m_sGenerationId;
    QFile::remove(sTemporaryPath);
    QFile::remove(sBackupPath);
    if(!nmDataJsonTools::WriteDomToFile(oDocument, sTemporaryPath) ||
       !flushFileToDisk(sTemporaryPath) ||
       !publishManifestFile(sManifestPath, sTemporaryPath, sBackupPath))
    {
        QFile::remove(sTemporaryPath);
        return setError(pError, "Cannot atomically publish numerical root manifest.");
    }

    // 发布后必须重新从磁盘解析和校验；失败时恢复原清单或删除无效的新清单。
    if(!validatePublishedManifest(m_sResultRootDirectory, pError))
    {
        const QString sValidationError = pError == NULL
                ? QString("Published numerical root manifest validation failed.")
                : *pError;
        bool bRolledBack = false;
        if(bHadPreviousManifest)
        {
            bRolledBack = restoreManifestBackup(
                    sManifestPath, sBackupPath) &&
                    validatePublishedManifest(m_sResultRootDirectory, NULL);
        }
        else
        {
            bRolledBack = QFile::remove(sManifestPath) ||
                    !QFileInfo(sManifestPath).exists();
        }
        if(!bRolledBack)
        {
            // 回滚失败时不能再由 abort 删除新代次，否则磁盘上的新清单会悬空。
            m_bKeepGenerationOnAbort = true;
            m_bActive = false;
            return setError(pError, sValidationError +
                    " Root manifest rollback failed; the new generation and backup were retained.");
        }
        return setError(pError, sValidationError);
    }
    QFile::remove(sBackupPath);
    m_bCommitted = true;
    m_bActive = false;

    // 只清理新清单未引用的代次；文件占用导致失败不影响已发布清单。
    // 空窗口清单仍保留一个空代次目录，供根清单发布后完整性复核。
    setReferencedGenerations.insert(m_sGenerationId);
    oIt = m_mapWindows.constBegin();
    for(; oIt != m_mapWindows.constEnd(); ++oIt)
    {
        const QStringList listParts = oIt.value().m_oMainJson.m_sRelativePath
                .split('/');
        if(listParts.size() >= 2 && listParts[0] == "NumericalGenerations")
        {
            setReferencedGenerations.insert(listParts[1]);
        }
    }
    QDir oGenerations(QDir(m_sResultRootDirectory).filePath(
            "NumericalGenerations"));
    const QFileInfoList listGenerations = oGenerations.entryInfoList(
            QDir::Dirs | QDir::NoDotAndDotDot);
    for(int nIndex = 0; nIndex < listGenerations.size(); ++nIndex)
    {
        if(!setReferencedGenerations.contains(listGenerations[nIndex].fileName()))
        {
            removeDirectoryTree(listGenerations[nIndex].absoluteFilePath());
        }
    }
    return true;
}

void nmNumericalSaveSession::abort()
{
    if(!m_bCommitted && !m_bKeepGenerationOnAbort &&
       !m_sGenerationDirectory.isEmpty())
    {
        removeDirectoryTree(m_sGenerationDirectory);
    }
    m_bActive = false;
    m_bCommitted = false;
    m_bKeepGenerationOnAbort = false;
    m_sResultId.clear();
    m_sResultRootDirectory.clear();
    m_sGenerationId.clear();
    m_sGenerationDirectory.clear();
    m_mapWindows.clear();
    m_setValidatedWindowIds.clear();
}

bool nmNumericalSaveSession::isActive() const
{
    return m_bActive;
}

QString nmNumericalSaveSession::generationId() const
{
    return m_sGenerationId;
}

QString nmNumericalSaveSession::resultRootDirectory() const
{
    return m_sResultRootDirectory;
}

bool nmNumericalSaveSession::resolveWindowMainJson(
        const QString& sResultRootDirectory,
        const QString& sWindowId,
        QString& sWindowDirectory,
        QString& sMainJsonPath,
        nmNumericalFileReference* pMainReference,
        QString* pError)
{
    sWindowDirectory.clear();
    sMainJsonPath.clear();
    QMap<QString, nmNumericalSavedWindow> mapWindows;
    if(!loadManifest(sResultRootDirectory, mapWindows, NULL, pError) ||
       !mapWindows.contains(sWindowId))
    {
        return setError(pError, "Numerical v3 manifest does not reference this window.");
    }
    const nmNumericalFileReference oReference =
            mapWindows.value(sWindowId).m_oMainJson;
    if(!nmNumericalResultPersistence::validateFileReference(
                sResultRootDirectory, oReference, &sMainJsonPath, pError))
    {
        return false;
    }
    sWindowDirectory = QFileInfo(sMainJsonPath).absolutePath();
    if(pMainReference != NULL)
    {
        *pMainReference = oReference;
    }
    return true;
}

bool nmNumericalSaveSession::containsPublishedWindow(
        const QString& sResultRootDirectory,
        const QString& sWindowId,
        bool& bContainsWindow,
        QString* pError)
{
    bContainsWindow = false;
    QString sNormalizedWindowId;
    if(!nmNumericalResultPersistence::normalizeRelativePath(
                sWindowId, sNormalizedWindowId) ||
       sNormalizedWindowId != sWindowId ||
       sNormalizedWindowId.contains('/'))
    {
        return setError(pError, "Numerical window identifier is invalid.");
    }

    QMap<QString, nmNumericalSavedWindow> mapWindows;
    if(!loadManifest(sResultRootDirectory, mapWindows, NULL, pError))
    {
        return false;
    }

    // 空清单或未引用当前窗口表示该分析没有数值成果，不属于加载错误。
    bContainsWindow = mapWindows.contains(sWindowId);
    return true;
}

bool nmNumericalSaveSession::validatePublishedManifest(
        const QString& sResultRootDirectory,
        QString* pError)
{
    QMap<QString, nmNumericalSavedWindow> mapWindows;
    return loadManifest(sResultRootDirectory, mapWindows, NULL, pError);
}