// Copyright Epic Games, Inc. All Rights Reserved.

#include "Commandlets/DiffAssetRegistriesCommandlet.h"

#include "AssetData.h"

#include "UObject/Class.h"
#include "PlatformInfo.h"
#include "Misc/Paths.h"
#include "HAL/FileManager.h"
#include "Serialization/ArrayReader.h"
#include "Misc/FileHelper.h"
#include "Internationalization/Regex.h"

DEFINE_LOG_CATEGORY(LogDiffAssets);

void UDiffAssetRegistriesCommandlet::PopulateChangelistMap(const FString& Branch, const FString& CL, bool bEnginePackages)
{
    FString FilePattern = FString::Printf(TEXT("%s%s@*.p4cache"), *FPaths::DiffDir(), *Branch);
    FString FileName    = FString::Printf(TEXT("%s%s@%s.p4cache"), *FPaths::DiffDir(), *Branch, *CL);

    TArray<FString> CacheFiles;

    IFileManager::Get().FindFiles(CacheFiles, *FilePattern, true, false);

    // search through the list of cache files for the newest one we can use
    int32 CLNum;
    LexTryParseString<int32>(CLNum, *CL);

    int32 BestCLNum = 0;
    FString BestCache;

    for (TArray<FString>::TConstIterator It(CacheFiles); It; ++It)
    {
        FString TestCL;
        FPaths::GetBaseFilename(*It).Split(TEXT("@"), nullptr, &TestCL);
        int32 TestCLNum;
        LexTryParseString<int32>(TestCLNum, *TestCL);

        if (TestCLNum <= CLNum && TestCLNum > BestCLNum)
        {
            BestCLNum = TestCLNum;
            BestCache = *It;
        }
    }

    FArchive* CacheFile = nullptr;

    // read in the baseline from the newest one
    if (BestCLNum > 0)
    {
        CacheFile = IFileManager::Get().CreateFileReader(*(FPaths::DiffDir() / BestCache));
        *CacheFile << AssetPathToChangelist;
        delete CacheFile;
    }

    FString CLRange = CL;

    // grab the newer file lists, or all of them if we had no best one, and merge them in
    if (BestCLNum < CLNum)
    {
        if (BestCLNum > 0)
            CLRange = FString::Printf(TEXT("%d,%d"), BestCLNum, CLNum);

        // skip the game packages if we're doing engine packages only
        if (!bEnginePackages)
        {
            FillChangelists(Branch, CLRange, P4GameBasePath, P4GameAssetPath);
        }
        FillChangelists(Branch, CLRange, P4EngineBasePath, P4EngineAssetPath);
    }

    // save out the new table
    if (BestCLNum != CLNum)
    {
        CacheFile = IFileManager::Get().CreateFileWriter(*FileName);
        *CacheFile << AssetPathToChangelist;

        delete CacheFile;
    }
}

int32 UDiffAssetRegistriesCommandlet::Main(const FString& FullCommandLine)
{
    UE_LOG(LogDiffAssets, Display, TEXT("--------------------------------------------------------------------------------------------"));
    UE_LOG(LogDiffAssets, Display, TEXT("Running DiffAssetRegistries Commandlet"));

    TArray<FString> Tokens;
    TArray<FString> Switches;
    TMap<FString, FString> Params;

    ParseCommandLine(*FullCommandLine, Tokens, Switches, Params);

    DiffChunkID = -1;
    bIsVerbose  = Switches.Contains(TEXT("VERBOSE"));
    {
        bSaveCSV = Switches.Contains(TEXT("CSV"));
        FString CSVName;
        bSaveCSV |= FParse::Value(*FullCommandLine, TEXT("CSVName="), CSVName);
        FString CSVPath;
        bSaveCSV |= FParse::Value(*FullCommandLine, TEXT("CSVPath="), CSVPath);

        if (bSaveCSV)
        {
            if (CSVFilename.IsEmpty() && !CSVName.IsEmpty())
            {
                CSVFilename = FString::Printf(TEXT("%s%s"), *FPaths::DiffDir(), *CSVName);
            }
            if (CSVFilename.IsEmpty())
            {
                CSVFilename = CSVPath;
            }
            if (CSVFilename.IsEmpty())
            {
                CSVFilename = FPaths::Combine(*FPaths::DiffDir(), TEXT("AssetChanges.csv"));
            }
            if (FPaths::GetExtension(CSVFilename).IsEmpty())
            {
                CSVFilename += TEXT(".csv");
            }
        }
    }

    // options to ignore small changes/sizes
    FParse::Value(*FullCommandLine, TEXT("MinChanges="), MinChangeCount);
    FParse::Value(*FullCommandLine, TEXT("MinChangeSize="), MinChangeSizeMB);
    FParse::Value(*FullCommandLine, TEXT("ChunkID="), DiffChunkID);
    FParse::Value(*FullCommandLine, TEXT("WarnPercentage="), WarnPercentage);
    FParse::Value(*FullCommandLine, TEXT("WarnSizeMin="), WarnSizeMinMB);
    FParse::Value(*FullCommandLine, TEXT("WarnTotalChangedSize="), WarnTotalChangedSizeMB);

    FString OldPath;
    FString NewPath;

    const bool bUseSourceGuid  = Switches.Contains(TEXT("SOURCEGUID"));
    const bool bConsistency    = Switches.Contains(TEXT("CONSISTENCY"));
    const bool bEnginePackages = Switches.Contains(TEXT("ENGINEPACKAGES"));
    bGroupByChunk              = Switches.Contains(TEXT("GROUPBYCHUNK"));

    FString SortOrder;
    FParse::Value(*FullCommandLine, TEXT("Sort="), SortOrder);

    if (SortOrder == TEXT("name"))
    {
        ReportedFileOrder = SortOrder::ByName;
    }
    else if (SortOrder == TEXT("size"))
    {
        ReportedFileOrder = SortOrder::BySize;
    }
    else if (SortOrder == TEXT("class"))
    {
        ReportedFileOrder = SortOrder::ByClass;
    }
    else if (SortOrder == TEXT("change"))
    {
        ReportedFileOrder = SortOrder::ByChange;
    }

    FString Branch;
    FString CL;
    FString Spec;

    FParse::Value(*FullCommandLine, TEXT("platform="), TargetPlatform);

    if (TargetPlatform.IsEmpty())
    {
        UE_LOG(LogDiffAssets, Error, TEXT("No platform specified on the commandline use \"-platform=<platform>\"."));
    }

    auto FindAssetRegistryPath = [&](const FString& PathVal, FString& OutPath)
    {
        for (const FString& SearchPath: AssetRegistrySearchPath)
        {
            FString FinalSearchPath = SearchPath;
            FinalSearchPath.ReplaceInline(TEXT("[buildversion]"), *PathVal);
            FinalSearchPath.ReplaceInline(TEXT("[platform]"), *TargetPlatform);
            if (IFileManager::Get().FileExists(*FinalSearchPath))
            {
                OutPath = FinalSearchPath;
                return true;
            }
        }
        return false;
    };

    // const TCHAR* AssetRegistrySubPath = TEXT("/Metadata/DevelopmentAssetRegistry.bin");
    const FString* OldPathVal = Params.Find(FString(TEXT("OldPath")));
    if (OldPathVal)
    {
        FindAssetRegistryPath(*OldPathVal, OldPath);
    }
    else
    {
        UE_LOG(LogDiffAssets, Error, TEXT("No old path specified \"-oldpath=<>\", use full path to asset registry or build version."));
        return -1;
    }

    const FString* NewPathVal = Params.Find(FString(TEXT("NewPath")));
    if (NewPathVal)
    {
        FindAssetRegistryPath(*NewPathVal, NewPath);

        if (!RegexBranchCL.IsEmpty())
        {
            const FRegexPattern CLPattern(RegexBranchCL);
            FRegexMatcher CLMatcher(CLPattern, NewPath);
            if (CLMatcher.FindNext())
            {
                Branch = CLMatcher.GetCaptureGroup(1);
                CL     = CLMatcher.GetCaptureGroup(2);
            }
        }
    }
    else
    {
        UE_LOG(LogDiffAssets, Error, TEXT("No new path specified \"-newpath=<>\", use full path to asset registry or build version."));
        return -1;
    }

    bMatchChangelists = false;
    if (FParse::Value(*FullCommandLine, TEXT("Branch="), Spec))
    {
        FString NewBranch, NewCL;

        bMatchChangelists = true;

        Spec.Split(TEXT("@"), &NewBranch, &NewCL);

        bMatchChangelists = true;

        PopulateChangelistMap(NewBranch, NewCL, bEnginePackages);
    }
    else if (Switches.Contains(TEXT("CHANGELISTS")))
    {
        bMatchChangelists = true;
        PopulateChangelistMap(Branch, CL, bEnginePackages);
    }

    if (OldPath.IsEmpty())
    {
        UE_LOG(LogDiffAssets, Error, TEXT("Unable to locate AssetRegistry.bin for supplied oldpath (%s), use full path to asset registry or build version."), **OldPathVal);
        return -1;
    }
    if (NewPath.IsEmpty())
    {
        UE_LOG(LogDiffAssets, Error, TEXT("Unable to locate AssetRegistry.bin for supplied newpath (%s), use full path to asset registry or build version."), **NewPathVal);
        return -1;
    }

    FPaths::NormalizeFilename(NewPath);
    FPaths::NormalizeFilename(OldPath);

    // try to discern platform
    /*if (NewPath.Contains(AssetRegistrySubPath))
    {
            FString NewPlatformDir = NewPath.Left(NewPath.Find(AssetRegistrySubPath));
            FString PlatformPath = FPaths::GetCleanFilename(NewPlatformDir);

            for (const FPlatformInfo& PlatformInfo : PlatformInfo::GetPlatformInfoArray())
            {
                    if (PlatformPath == PlatformInfo.TargetPlatformName.ToString())
                    {
                            TargetPlatform = PlatformPath;
                            break;
                    }
            }
    }*/

    if (bConsistency)
    {
        ConsistencyCheck(OldPath, NewPath);
    }
    else
    {
        DiffAssetRegistries(OldPath, NewPath, bUseSourceGuid, bEnginePackages);
    }
    UE_LOG(LogDiffAssets, Display, TEXT("Successfully finished running DiffAssetRegistries Commandlet"));
    UE_LOG(LogDiffAssets, Display, TEXT("--------------------------------------------------------------------------------------------"));
    return 0;
}

void UDiffAssetRegistriesCommandlet::FillChangelists(FString Branch, FString CL, FString BasePath, FString AssetPath)
{
    TArray<FString> Results;
    int32 ReturnCode = 0;
    if (LaunchP4(TEXT("files ") + P4Repository + Branch + BasePath + TEXT("....uasset@") + CL, Results, ReturnCode))
    {
        if (ReturnCode == 0)
        {
            for (const FString& Result: Results)
            {
                FString DepotPathName;
                FString ExtraInfoAfterPound;
                if (Result.Split(TEXT("#"), &DepotPathName, &ExtraInfoAfterPound))
                {
                    FString PostContentPath;
                    if (DepotPathName.Split(BasePath, nullptr, &PostContentPath))
                    {
                        if (!PostContentPath.IsEmpty() && !PostContentPath.StartsWith(TEXT("Cinematics")) && !PostContentPath.StartsWith(TEXT("Developers")) && !PostContentPath.StartsWith(TEXT("Maps/Test_Maps")))
                        {
                            const FString PostContentPathWithoutExtension = FPaths::GetBaseFilename(PostContentPath, false);
                            const FString FullPackageName                 = AssetPath + PostContentPathWithoutExtension;

                            TArray<FString> Chunks;

                            ExtraInfoAfterPound.ParseIntoArray(Chunks, TEXT(" "), true);

                            int32 Changelist;

                            LexTryParseString<int32>(Changelist, *Chunks[4]);
                            if (Changelist)
                            {
                                int32& Entry = AssetPathToChangelist.FindOrAdd(*FullPackageName);

                                if (Changelist > Entry)
                                    Entry = Changelist;
                            }
                        }
                    }
                }
            }
        }
    }
    if (LaunchP4(TEXT("files ") + P4Repository + Branch + BasePath + TEXT("....umap@") + CL, Results, ReturnCode))
    {
        if (ReturnCode == 0)
        {
            for (const FString& Result: Results)
            {
                FString DepotPathName;
                FString ExtraInfoAfterPound;
                if (Result.Split(TEXT("#"), &DepotPathName, &ExtraInfoAfterPound))
                {
                    FString PostContentPath;
                    if (DepotPathName.Split(BasePath, nullptr, &PostContentPath))
                    {
                        if (!PostContentPath.IsEmpty() && !PostContentPath.StartsWith(TEXT("Cinematics")) && !PostContentPath.StartsWith(TEXT("Developers")) && !PostContentPath.StartsWith(TEXT("Maps/Test_Maps")))
                        {
                            const FString PostContentPathWithoutExtension = FPaths::GetBaseFilename(PostContentPath, false);
                            const FString FullPackageName                 = AssetPath + PostContentPathWithoutExtension;

                            TArray<FString> Chunks;

                            ExtraInfoAfterPound.ParseIntoArray(Chunks, TEXT(" "), true);

                            int32 Changelist;

                            LexTryParseString<int32>(Changelist, *Chunks[4]);
                            if (Changelist)
                            {
                                int32& Entry = AssetPathToChangelist.FindOrAdd(*FullPackageName);

                                if (Changelist > Entry)
                                    Entry = Changelist;
                            }
                        }
                    }
                }
            }
        }
    }
}

void rescale(int bytes, double& value, int& exp)
{
    value = bytes;
    value = fabs(value);

    while (value > 1024.0)
    {
        value /= 1024.0;
        exp++;
    }
}

void UDiffAssetRegistriesCommandlet::ConsistencyCheck(const FString& OldPath, const FString& NewPath)
{
    {
        FArrayReader SerializedAssetData;

        if (!IFileManager::Get().FileExists(*OldPath))
        {
            UE_LOG(LogDiffAssets, Error, TEXT("File '%s' does not exist."), *OldPath);
            return;
        }
        if (!FFileHelper::LoadFileToArray(SerializedAssetData, *OldPath))
        {
            UE_LOG(LogDiffAssets, Error, TEXT("Failed to load file '%s'."), *OldPath);
            return;
        }
        if (!OldState.Load(SerializedAssetData))
        {
            UE_LOG(LogDiffAssets, Error, TEXT("Failed to parse file '%s' as asset registry."), *OldPath);
            return;
        }
    }
    {
        FArrayReader SerializedAssetData;

        if (!IFileManager::Get().FileExists(*NewPath))
        {
            UE_LOG(LogDiffAssets, Error, TEXT("File '%s' does not exist."), *NewPath);
            return;
        }
        if (!FFileHelper::LoadFileToArray(SerializedAssetData, *NewPath))
        {
            UE_LOG(LogDiffAssets, Error, TEXT("Failed to load file '%s'."), *NewPath);
            return;
        }

        if (!OldState.Load(SerializedAssetData))
        {
            UE_LOG(LogDiffAssets, Error, TEXT("Failed to parse file '%s' as asset registry."), *NewPath);
            return;
        }
    }
    UE_LOG(LogDiffAssets, Display, TEXT("Comparing asset registries '%s' and '%s'."), *OldPath, *NewPath);
    UE_LOG(LogDiffAssets, Display, TEXT("Source vs Cooked Consistency Diff"));
    if (bIsVerbose)
    {
        UE_LOG(LogDiffAssets, Display, TEXT("Cooked files that differ, where source guids do not:"));
    }
    // We're looking for packages that the Cooked check says are modified, but that the Guid check says are not
    // We're ignoring new packages for this, as those are obviously going to change
    TSet<FName> GuidModified, CookModified;
    TSet<FName> New;

    for (const TPair<FName, const FAssetPackageData*>& Pair: NewState.GetAssetPackageDataMap())
    {
        FName Name                        = Pair.Key;
        const FAssetPackageData* Data     = Pair.Value;
        const FAssetPackageData* PrevData = OldState.GetAssetPackageData(Name);

        if (!PrevData)
        {
            New.Add(Name);
        }
        else
        {
            PRAGMA_DISABLE_DEPRECATION_WARNINGS
            if (Data->PackageGuid != PrevData->PackageGuid)
                PRAGMA_ENABLE_DEPRECATION_WARNINGS
                {
                    GuidModified.Add(Name);
                }
            if (Data->CookedHash != PrevData->CookedHash)
            {
                CookModified.Add(Name);
            }
        }
    }

    // recurse through the referencer lists to fill out GuidModified
    TArray<FName> Recurse = GuidModified.Array();

    for (int32 RecurseIndex = 0; RecurseIndex < Recurse.Num(); RecurseIndex++)
    {
        FName Package = Recurse[RecurseIndex];
        TArray<FAssetIdentifier> Referencers;
        NewState.GetReferencers(Package, Referencers, UE::AssetRegistry::EDependencyCategory::Package, UE::AssetRegistry::EDependencyQuery::Hard);

        for (const FAssetIdentifier& Referencer: Referencers)
        {
            FName ReferencerPackage = Referencer.PackageName;
            if (!New.Contains(ReferencerPackage) && !GuidModified.Contains(ReferencerPackage))
            {
                GuidModified.Add(ReferencerPackage);
                Recurse.Add(ReferencerPackage);
            }
        }
    }

    int64 changes      = 0;
    int64 change_bytes = 0;

    // find all entries of CookModified that do not exist in GuidModified
    const TMap<FName, const FAssetPackageData*>& PackageMap = NewState.GetAssetPackageDataMap();
    for (FName const& Package: CookModified)
    {
        const FAssetPackageData* Data = PackageMap[Package];

        if (!GuidModified.Contains(Package))
        {
            ++changes;
            change_bytes += Data->DiskSize;
            if (bIsVerbose)
            {
                UE_LOG(LogDiffAssets, Display, TEXT("%s : %d bytes"), *Package.ToString(), Data->DiskSize);
            }
        }
    }

    double change_value = 0.0;
    int change_exp      = 0;

    rescale(change_bytes, change_value, change_exp);

    UE_LOG(LogDiffAssets, Display, TEXT("Summary:"));
    UE_LOG(LogDiffAssets, Display, TEXT("%d nondeterministic cooks, %8.3f %cB"), changes, change_value, " KMGTP"[change_exp]);
}

bool UDiffAssetRegistriesCommandlet::IsInRelevantChunk(FAssetRegistryState& InRegistryState, FName InAssetPath)
{
    if (DiffChunkID == -1)
    {
        return true;
    }
    TArrayView<FAssetData const* const> Assets = InRegistryState.GetAssetsByPackageName(InAssetPath);

    if (Assets.Num() && Assets[0]->ChunkIDs.Num())
    {
        return Assets[0]->ChunkIDs.Contains(DiffChunkID);
    }

    return true;
}

FName UDiffAssetRegistriesCommandlet::GetClassName(FAssetRegistryState& InRegistryState, FName InAssetPath)
{
    if (AssetPathToClassName.Contains(InAssetPath) == false)
    {
        TArrayView<FAssetData const* const> Assets = InRegistryState.GetAssetsByPackageName(InAssetPath);

        FName NewName                              = NAME_None;
        if (Assets.Num() > 0)
        {
            NewName = Assets[0]->AssetClass;
        }
        else
        {
            if (InAssetPath.ToString().StartsWith(TEXT("/Script/")))
            {
                NewName = NAME_Class;
            }
        }

        if (NewName == NAME_None)
        {
            UE_LOG(LogDiffAssets, Log, TEXT("Unable to find class type of asset %s"), *InAssetPath.ToString());
        }
        AssetPathToClassName.Add(InAssetPath) = NewName;
    }

    return AssetPathToClassName[InAssetPath];
}

TArray<int32> UDiffAssetRegistriesCommandlet::GetAssetChunks(FAssetRegistryState& InRegistryState, FName InAssetPath)
{
    if (ChunkIdByAssetPath.Contains(InAssetPath) == false)
    {
        TArrayView<FAssetData const* const> Assets = InRegistryState.GetAssetsByPackageName(InAssetPath);

        if (Assets.Num() > 0 && Assets[0]->ChunkIDs.Num() > 0)
        {
            if (Assets[0]->ChunkIDs.Num() > 1)
            {
                UE_LOG(LogDiffAssets, Log, TEXT("Multiple ChunkIds for asset %s"), *InAssetPath.ToString());
            }

            for (int32 id: Assets[0]->ChunkIDs)
            {
                ChangesByChunk.FindOrAdd(id).IncludedAssets.Add(InAssetPath);
                ChunkIdByAssetPath.Add(InAssetPath, id);
            }
        }
        else
        {
            UE_LOG(LogDiffAssets, Log, TEXT("Unable to find chunk ids of asset %s"), *InAssetPath.ToString());
            ChunkIdByAssetPath.Add(InAssetPath, -1);
        }
    }

    TArray<int32> ChunkIds;
    ChunkIdByAssetPath.MultiFind(InAssetPath, ChunkIds);
    return ChunkIds;
}

void UDiffAssetRegistriesCommandlet::RecordAdd(FName InAssetPath, const FAssetPackageData& InNewData)
{
    FChangeInfo AssetChange;

    ++AssetChange.Adds;
    if (InNewData.DiskSize > 0)
    {
        AssetChange.AddedBytes += InNewData.DiskSize;
    }

    FName ClassName                          = GetClassName(NewState, InAssetPath);
    TArray<int32> ChunkIds                   = GetAssetChunks(NewState, InAssetPath);

    int changelist                           = AssetPathToChangelist.FindOrAdd(*InAssetPath.ToString());

    ChangeInfoByAsset.FindOrAdd(InAssetPath) = AssetChange;
    ChangeSummaryByClass.FindOrAdd(ClassName) += AssetChange;
    ChangeSummaryByChangelist.FindOrAdd(changelist) += AssetChange;
    for (int32 ChunkId: ChunkIds)
    {
        ChangesByChunk.FindOrAdd(ChunkId).ChangesByClass.FindOrAdd(ClassName) += AssetChange;
    }
    ChangeSummary += AssetChange;
}

void UDiffAssetRegistriesCommandlet::RecordEdit(FName InAssetPath, const FAssetPackageData& InNewData, const FAssetPackageData& InOldData)
{
    FChangeInfo AssetChange;

    if (InNewData.DiskSize > 0)
    {
        ++AssetChange.Changes;
        AssetChange.ChangedBytes += InNewData.DiskSize;
    }

    FName ClassName                          = GetClassName(NewState, InAssetPath);
    TArray<int32> ChunkIds                   = GetAssetChunks(NewState, InAssetPath);

    int changelist                           = AssetPathToChangelist.FindOrAdd(*InAssetPath.ToString());

    ChangeInfoByAsset.FindOrAdd(InAssetPath) = AssetChange;
    ChangeSummaryByClass.FindOrAdd(ClassName) += AssetChange;
    ChangeSummaryByChangelist.FindOrAdd(changelist) += AssetChange;
    for (int32 ChunkId: ChunkIds)
    {
        ChangesByChunk.FindOrAdd(ChunkId).ChangesByClass.FindOrAdd(ClassName) += AssetChange;
    }
    ChangeSummary += AssetChange;
}

void UDiffAssetRegistriesCommandlet::RecordDelete(FName InAssetPath, const FAssetPackageData& InData)
{
    FChangeInfo AssetChange;

    ++AssetChange.Deletes;

    if (InData.DiskSize >= 0)
    {
        AssetChange.DeletedBytes += InData.DiskSize;
    }

    FName ClassName                          = GetClassName(OldState, InAssetPath);
    TArray<int32> ChunkIds                   = GetAssetChunks(NewState, InAssetPath);

    ChangeInfoByAsset.FindOrAdd(InAssetPath) = AssetChange;
    ChangeSummaryByClass.FindOrAdd(ClassName) += AssetChange;
    for (int32 ChunkId: ChunkIds)
    {
        ChangesByChunk.FindOrAdd(ChunkId).ChangesByClass.FindOrAdd(ClassName) += AssetChange;
    }
    ChangeSummary += AssetChange;
}

void UDiffAssetRegistriesCommandlet::RecordNoChange(FName InAssetPath, const FAssetPackageData& InData)
{
    FChangeInfo AssetChange;

    AssetChange.Unchanged++;

    if (InData.DiskSize >= 0)
    {
        AssetChange.UnchangedBytes += InData.DiskSize;
    }

    FName ClassName                          = GetClassName(NewState, InAssetPath);
    TArray<int32> ChunkIds                   = GetAssetChunks(NewState, InAssetPath);

    ChangeInfoByAsset.FindOrAdd(InAssetPath) = AssetChange;
    ChangeSummaryByClass.FindOrAdd(ClassName) += AssetChange;
    for (int32 ChunkId: ChunkIds)
    {
        ChangesByChunk.FindOrAdd(ChunkId).ChangesByClass.FindOrAdd(ClassName) += AssetChange;
    }
    ChangeSummary += AssetChange;
}

void UDiffAssetRegistriesCommandlet::SummarizeDeterminism()
{
    TArray<FName> AssetPaths;
    ChangeInfoByAsset.GetKeys(AssetPaths);

    for (const FName& AssetPath: AssetPaths)
    {
        const FChangeInfo& ChangeInfo = ChangeInfoByAsset[AssetPath];

        char classification;

        // classify the asset change by the flags
        int32 flags = AssetPathFlags.FindOrAdd(AssetPath);
        {
            bool hash    = (flags & EAssetFlags::HashChange) != 0;
            bool guid    = (flags & EAssetFlags::GuidChange) != 0;
            bool dephash = (flags & EAssetFlags::DepHashChange) != 0;
            bool depguid = (flags & EAssetFlags::DepGuidChange) != 0;

            if (!hash)
                classification = 'x';  // shouldn't see this in here, no binary change
            else
            {
                if (guid)
                    classification = 'e';  // explicit edit
                else if (dephash & depguid)
                    classification = 'd';  // dependency edit
                else if (dephash & !depguid)
                    classification = 'n';  // nondeterministic dependency
                else
                    classification = 'c';  // nondeterministic
            }
        }

        TArray<int32> ChunkIds = GetAssetChunks(NewState, AssetPath);
        FName ClassName        = GetClassName(NewState, AssetPath);
        if (classification == 'c')
        {
            NondeterministicSummary += ChangeInfo;

            for (int32 ChunkId: ChunkIds)
            {
                ChangesByChunk.FindOrAdd(ChunkId).Determinism.FindOrAdd(ClassName).AddDirect(ChangeInfo);
            }
            DeterminismByClass.FindOrAdd(ClassName).AddDirect(ChangeInfo);
        }
        else if (classification == 'n')
        {
            IndirectNondeterministicSummary += ChangeInfo;

            for (int32 ChunkId: ChunkIds)
            {
                ChangesByChunk.FindOrAdd(ChunkId).Determinism.FindOrAdd(ClassName).AddIndirect(ChangeInfo);
            }
            DeterminismByClass.FindOrAdd(ClassName).AddIndirect(ChangeInfo);
        }
    }
}

void UDiffAssetRegistriesCommandlet::LogChangedFiles(FArchive* CSVFile, FString const& OldPath, FString const& NewPath)
{
    if (!bIsVerbose && !bSaveCSV)
    {
        return;
    }

    TArray<FName> AssetPaths;
    ChangeInfoByAsset.GetKeys(AssetPaths);

    // sort by size
    if (ReportedFileOrder == SortOrder::BySize)
    {
        // Sort by size of change
        AssetPaths.Sort([this](const FName& Lhs, const FName& Rhs)
                        {
                            return ChangeInfoByAsset[Lhs].GetTotalChangeSize() > ChangeInfoByAsset[Rhs].GetTotalChangeSize();
                        });
    }
    // Sort by class type then size size
    else if (ReportedFileOrder == SortOrder::ByClass)
    {
        AssetPaths.Sort([this](const FName& Lhs, const FName& Rhs)
                        {
                            FString LhsName = GetClassName(NewState, Lhs).ToString();
                            FString RhsName = GetClassName(NewState, Rhs).ToString();

                            if (LhsName != RhsName)
                            {
                                return LhsName < RhsName;
                            }

                            return ChangeInfoByAsset[Lhs].GetTotalChangeSize() > ChangeInfoByAsset[Rhs].GetTotalChangeSize();
                        });
    }
    // sort by change type then size
    else if (ReportedFileOrder == SortOrder::ByChange)
    {
        AssetPaths.Sort([this](const FName& Lhs, const FName& Rhs)
                        {
                            int32 LHSChanges = ChangeInfoByAsset[Lhs].GetChangeFlags();
                            int32 RHSChanges = ChangeInfoByAsset[Rhs].GetChangeFlags();

                            if (LHSChanges != RHSChanges)
                            {
                                return LHSChanges > RHSChanges;
                            }

                            // sort by size
                            return ChangeInfoByAsset[Lhs].GetTotalChangeSize() > ChangeInfoByAsset[Rhs].GetTotalChangeSize();
                        });
    }
    // sort by name
    else if (ReportedFileOrder == SortOrder::ByName)
    {
        AssetPaths.Sort([this](const FName& Lhs, const FName& Rhs)
                        {
                            return Lhs.ToString() < Rhs.ToString();
                        });
    }

    if (CSVFile)
    {
        CSVFile->Logf(TEXT("Type Key"));
        CSVFile->Logf(TEXT("a, file added"));
        CSVFile->Logf(TEXT("r, file removed"));
        CSVFile->Logf(TEXT("e, explicit edit (this file specifically has been modified)"));
        CSVFile->Logf(TEXT("d, dependency edit (this file is different likely because a dependency has also been changed)"));
        CSVFile->Logf(TEXT("n, indirect non deterministic (a dependency file changed but wasn't changed directly (Indicates the dependency was either non determinisitc or another indirect non deterministic file))"));
        CSVFile->Logf(TEXT("c, non deterministic (the hashes for all dependencies are the same but this file is not)"));
        CSVFile->Logf(TEXT("x, no binary change (shouldn't ever happen)"));
        CSVFile->Logf(TEXT(""));

        CSVFile->Logf(TEXT("Modification,Name,Class,NewSize,OldSize,Changelist,Chunk"));

        UE_LOG(LogDiffAssets, Display, TEXT("Saving CSV results to %s"), *CSVFilename);
    }

    for (const FName& AssetPath: AssetPaths)
    {
        const FChangeInfo& ChangeInfo = ChangeInfoByAsset[AssetPath];

        int Changelist                = bMatchChangelists ? AssetPathToChangelist.FindOrAdd(*AssetPath.ToString()) : 0;

        FName ClassName;

        if (ChangeInfo.Deletes)
        {
            ClassName = GetClassName(OldState, AssetPath);
        }
        else
        {
            ClassName = GetClassName(NewState, AssetPath);
        }
        auto GetChunkIDString = [this, AssetPath = AssetPath](FAssetRegistryState& State)
        {
            TArray<int32> ChunkIds = GetAssetChunks(State, AssetPath);
            FString ChunkIdString  = FString::JoinBy(ChunkIds, TEXT(" & "), [](int32 Value)
                                                     {
                                                        return FString::Printf(TEXT("%d"), Value);
                                                    });
            return ChunkIdString;
        };

        if (ChangeInfo.Adds)
        {
            if (CSVFile)
            {
                CSVFile->Logf(TEXT("a,%s,%s,%d,0,%d,%s"), *AssetPath.ToString(), *ClassName.ToString(), ChangeInfo.AddedBytes, Changelist, *GetChunkIDString(NewState));
            }

            if (bIsVerbose)
            {
                UE_LOG(LogDiffAssets, Display, TEXT("a %s : (Class=%s,NewSize=%d bytes)"), *AssetPath.ToString(), *ClassName.ToString(), ChangeInfo.AddedBytes);
            }
        }
        else if (ChangeInfo.Changes)
        {
            const FAssetPackageData* PrevData = OldState.GetAssetPackageData(AssetPath);

            char classification;

            // classify the asset change by the flags
            int32 flags = AssetPathFlags.FindOrAdd(AssetPath);
            {
                bool hash    = (flags & EAssetFlags::HashChange) != 0;
                bool guid    = (flags & EAssetFlags::GuidChange) != 0;
                bool dephash = (flags & EAssetFlags::DepHashChange) != 0;
                bool depguid = (flags & EAssetFlags::DepGuidChange) != 0;

                if (!hash)
                    classification = 'x';  // shouldn't see this in here, no binary change
                else
                {
                    if (guid)
                        classification = 'e';  // explicit edit
                    else if (dephash & depguid)
                        classification = 'd';  // dependency edit
                    else if (dephash & !depguid)
                        classification = 'n';  // nondeterministic dependency
                    else
                        classification = 'c';  // nondeterministic
                }
            }

            if (CSVFile)
            {
                CSVFile->Logf(TEXT("%c,%s,%s,%d,%d,%d,%s"), classification, *AssetPath.ToString(), *ClassName.ToString(), ChangeInfo.ChangedBytes, PrevData->DiskSize, Changelist, *GetChunkIDString(NewState));
            }

            if (bIsVerbose)
            {
                UE_LOG(LogDiffAssets, Display, TEXT("%c %s : (Class=%s,NewSize=%d bytes,OldSize=%d bytes)"), classification, *AssetPath.ToString(), *ClassName.ToString(), ChangeInfo.ChangedBytes, PrevData->DiskSize);
            }
            // UE_LOG(LogDiffAssets, Display, TEXT("Source file changed: %s"), (flags & EAssetFlags::GuidChange) ? TEXT("true") : TEXT("false"));
            /*if ((flags & EAssetFlags::GuidChange) && bMatchChangelists)
            {
                    UE_LOG(LogDiffAssets, Display, TEXT("Last change: %d"), Changelist);
            }*/
#if 0			
			TArray<FAssetIdentifier> Dependencies;
			NewState.GetDependencies(AssetPath, Dependencies, EAssetRegistryDependencyType::Hard);
			if (Dependencies.Num() > 0)
			{
				UE_LOG(LogDiffAssets, Display, TEXT("Dependency changes:"));

				for (const FAssetIdentifier& Dependency : Dependencies)
				{
					if (AssetPathToSourceChanged.FindOrAdd(Dependency.PackageName))
					{
						UE_LOG(LogDiffAssets, Display, TEXT("  %s: %d"), *Dependency.PackageName.ToString(), AssetPathToChangelist.FindOrAdd(Dependency.PackageName));
					}
				}
			}
#endif
        }
        else if (ChangeInfo.Deletes)
        {
            const FAssetPackageData* PrevData = OldState.GetAssetPackageData(AssetPath);

            if (CSVFile)
            {
                CSVFile->Logf(TEXT("r,%s,%s,0,%d,0,0"), *AssetPath.ToString(), *ClassName.ToString(), PrevData->DiskSize);
            }

            if (bIsVerbose)
            {
                UE_LOG(LogDiffAssets, Display, TEXT("r %s : (Class=%s,OldSize=%d bytes)"), *AssetPath.ToString(), *ClassName.ToString(), PrevData->DiskSize);
            }
        }
    }
}

void UDiffAssetRegistriesCommandlet::LogClassSummary(FArchive* CSVFile, const FString& HeaderPrefix, const TMap<FName, FChangeInfo>& InChangeInfoByAsset, bool bDoWarnings, TMap<FName, FDeterminismInfo> DeterminismInfo)
{
    const float InvToMB  = 1.0 / (1024 * 1024);
    FString HeaderSpacer = (HeaderPrefix.Len() ? TEXT(" ") : TEXT(""));

    // show class totals first
    TArray<FName> ClassNames;
    InChangeInfoByAsset.GetKeys(ClassNames);

    // Sort keys by the desired order
    if (ReportedFileOrder == SortOrder::ByName || ReportedFileOrder == SortOrder::ByClass)
    {
        ClassNames.Sort([](const FName& Lhs, const FName& Rhs)
                        {
                            return Lhs.ToString() < Rhs.ToString();
                        });
    }
    else  // Default to size for everything else for class list
    {
        // sort by size of changes (number can also be a big impact on patch size but depends on datalayout and patch algo..)
        ClassNames.Sort([this, InChangeInfoByAsset](const FName& Lhs, const FName& Rhs)
                        {
                            const FChangeInfo& LHSChanges = InChangeInfoByAsset[Lhs];
                            const FChangeInfo& RHSChanges = InChangeInfoByAsset[Rhs];
                            return LHSChanges.GetTotalChangeSize() > RHSChanges.GetTotalChangeSize();
                        });
    }

    // Overall Class Summary
    bool bChangesPastThreshold = false;
    for (FName ClassName: ClassNames)
    {
        const FChangeInfo& Changes = InChangeInfoByAsset[ClassName];

        if (Changes.GetTotalChangeSize() == 0)
        {
            continue;
        }

        if (Changes.GetTotalChangeCount() < MinChangeCount || Changes.GetTotalChangeSize() < (MinChangeSizeMB * 1024 * 1024))
        {
            continue;
        }
        else if (!bChangesPastThreshold)
        {
            // We'll need to display the header, since this is the first row that has sufficient changes
            if (CSVFile)
            {
                FString ExtraNondeterminismHeader = TEXT("");
                if (DeterminismInfo.Num())
                {
                    ExtraNondeterminismHeader = TEXT(",DirectNondeterministicCount,DirectNondeterministicSize,IndirectNondeterministicCount,IndirectNondeterministicSize");
                }

                CSVFile->Logf(TEXT(""));
                CSVFile->Logf(TEXT("%s%sClass Summary"), *HeaderPrefix, *HeaderSpacer);
                CSVFile->Logf(TEXT("Name,Percentage,TotalCount,TotalSize,Adds,AddedSize,Changes,ChangesSize,Deletes,DeletedSize,Unchanged,UnchangedSize%s"), *ExtraNondeterminismHeader);
            }

            bChangesPastThreshold = true;
        }

        if (CSVFile)
        {
            FString ExtraNondeterminismData = TEXT("");
            if (DeterminismInfo.Num())
            {
                int64 DirectCount   = 0;
                int64 IndirectCount = 0;

                int64 DirectSize    = 0;
                int64 IndirectSize  = 0;
                if (DeterminismInfo.Contains(ClassName))
                {
                    int64 TotalCount = Changes.GetTotalChangeCount();
                    DirectCount      = DeterminismInfo[ClassName].DirectCount;
                    IndirectCount    = DeterminismInfo[ClassName].IndirectCount;

                    DirectSize       = DeterminismInfo[ClassName].DirectSize;
                    IndirectSize     = DeterminismInfo[ClassName].IndirectSize;
                }

                ExtraNondeterminismData = FString::Printf(TEXT(",%lld,%lld,%lld,%lld"),
                                                          DirectCount, DirectSize, IndirectCount, IndirectSize);
            }

            CSVFile->Logf(TEXT("%s,%0.02f,%lld,%lld,%lld,%lld,%lld,%lld,%lld,%lld,%lld,%lld%s"),
                          *ClassName.ToString(),
                          Changes.GetChangePercentage() * 100.0,
                          Changes.GetTotalChangeCount(),
                          Changes.GetTotalChangeSize(),
                          Changes.Adds, Changes.AddedBytes,
                          Changes.Changes, Changes.ChangedBytes,
                          Changes.Deletes, Changes.DeletedBytes,
                          Changes.Unchanged, Changes.UnchangedBytes,
                          *ExtraNondeterminismData);
        }

        // log summary & change
        UE_LOG(LogDiffAssets, Display, TEXT("%s%sClass Summary: "), *HeaderPrefix, *HeaderSpacer);

        UE_LOG(LogDiffAssets, Display, TEXT("%s: %.02f%% changes (%.02f MB Total)"),
               *ClassName.ToString(), Changes.GetChangePercentage() * 100.0, Changes.GetTotalChangeSize() * InvToMB);

        if (Changes.Adds)
        {
            UE_LOG(LogDiffAssets, Display, TEXT("\t%d packages added,    %8.3f MB"), Changes.Adds, Changes.AddedBytes * InvToMB);
        }

        if (Changes.Changes)
        {
            UE_LOG(LogDiffAssets, Display, TEXT("\t%d packages modified, %8.3f MB"), Changes.Changes, Changes.ChangedBytes * InvToMB);
        }

        if (Changes.Deletes)
        {
            UE_LOG(LogDiffAssets, Display, TEXT("\t%d packages removed,  %8.3f MB"), Changes.Deletes, Changes.DeletedBytes * InvToMB);
        }

        UE_LOG(LogDiffAssets, Display, TEXT("\t%d packages unchanged,  %8.3f MB"), Changes.Unchanged, Changes.UnchangedBytes * InvToMB);

        // Warn on a certain % of changes if that's enabled
        if (bDoWarnings && Changes.Changes >= 10 && (WarnPercentage > 0 || WarnSizeMinMB > 0) && Changes.ChangedBytes * InvToMB >= WarnSizeMinMB && Changes.GetChangePercentage() * 100.0 > WarnPercentage)
        {
            UE_LOG(LogDiffAssets, Warning, TEXT("\t%s Assets for %s are %.02f%% changed. (%.02f MB of data)"),
                   *TargetPlatform, *ClassName.ToString(), Changes.GetChangePercentage() * 100.0, Changes.ChangedBytes * InvToMB);
        }
    }

    // If we didn't find any changes of sufficient size, note as much instead of the header
    if (!bChangesPastThreshold)
    {
        FString SummaryName = (HeaderPrefix.Len() ? FString::Printf(TEXT("%s: "), *HeaderPrefix) : TEXT(""));

        if (CSVFile)
        {
            CSVFile->Logf(TEXT(""));
            CSVFile->Logf(TEXT("%sNo classes had changes past change threshold"), *SummaryName);
        }

        UE_LOG(LogDiffAssets, Display, TEXT("%sNo classes had changes past thresholds"), *SummaryName);
    }
}

void UDiffAssetRegistriesCommandlet::DiffAssetRegistries(const FString& OldPath, const FString& NewPath, bool bUseSourceGuid, bool bEnginePackagesOnly)
{
    {
        FArrayReader SerializedAssetData;

        if (!IFileManager::Get().FileExists(*OldPath))
        {
            UE_LOG(LogDiffAssets, Error, TEXT("File '%s' does not exist."), *OldPath);
            return;
        }
        if (!FFileHelper::LoadFileToArray(SerializedAssetData, *OldPath))
        {
            UE_LOG(LogDiffAssets, Error, TEXT("Failed to load file '%s'."), *OldPath);
            return;
        }
        if (!OldState.Load(SerializedAssetData))
        {
            UE_LOG(LogDiffAssets, Error, TEXT("Failed to parse file '%s' as asset registry."), *OldPath);
            return;
        }
    }
    {
        FArrayReader SerializedAssetData;

        if (!IFileManager::Get().FileExists(*NewPath))
        {
            UE_LOG(LogDiffAssets, Error, TEXT("File '%s' does not exist."), *NewPath);
            return;
        }
        if (!FFileHelper::LoadFileToArray(SerializedAssetData, *NewPath))
        {
            UE_LOG(LogDiffAssets, Error, TEXT("Failed to load file '%s'."), *NewPath);
            return;
        }
        if (!NewState.Load(SerializedAssetData))
        {
            UE_LOG(LogDiffAssets, Error, TEXT("Failed to parse file '%s' as asset registry."), *NewPath);
            return;
        }
    }

    int64 newtotal    = 0;
    int64 oldtotal    = 0;
    int64 newuncooked = 0;
    int64 olduncooked = 0;
    int64 newassets   = 0;
    int64 oldassets   = 0;

    UE_LOG(LogDiffAssets, Display, TEXT("Comparing asset registries '%s' and '%s'."), *OldPath, *NewPath);
    if (bUseSourceGuid)
    {
        UE_LOG(LogDiffAssets, Display, TEXT("Source Package Diff"));
    }
    else
    {
        UE_LOG(LogDiffAssets, Display, TEXT("Cooked Package Diff"));
    }
    if (bIsVerbose)
    {
        UE_LOG(LogDiffAssets, Display, TEXT("Package changes:"));
    }

    TSet<FName> Modified;
    TSet<FName> New;

    if (bUseSourceGuid)
    {
        for (const TPair<FName, const FAssetPackageData*>& Pair: NewState.GetAssetPackageDataMap())
        {
            FName Name         = Pair.Key;

            FString NameString = Name.ToString();

            if (bEnginePackagesOnly && !NameString.StartsWith(TEXT("/Engine/")))
            {
                continue;
            }

            if (!IsInRelevantChunk(NewState, Name))
            {
                continue;
            }

            const FAssetPackageData* Data     = Pair.Value;
            const FAssetPackageData* PrevData = OldState.GetAssetPackageData(Name);

            if (Data->DiskSize < 0)
            {
                newuncooked++;
            }

            newassets += NewState.GetAssetsByPackageName(Name).Num();

            if (!PrevData)
            {
                New.Add(Name);
                RecordAdd(Name, *Data);
            }
            PRAGMA_DISABLE_DEPRECATION_WARNINGS
            else if (Data->PackageGuid != PrevData->PackageGuid)
                PRAGMA_ENABLE_DEPRECATION_WARNINGS
            {
                Modified.Add(Name);
            }
            else
            {
                RecordNoChange(Name, *Data);
            }
            ++newtotal;
        }

        TArray<FName> Recurse = Modified.Array();

        for (int32 RecurseIndex = 0; RecurseIndex < Recurse.Num(); RecurseIndex++)
        {
            FName Package = Recurse[RecurseIndex];
            TArray<FAssetIdentifier> Referencers;
            NewState.GetReferencers(Package, Referencers, UE::AssetRegistry::EDependencyCategory::Package, UE::AssetRegistry::EDependencyQuery::Hard);

            for (const FAssetIdentifier& Referencer: Referencers)
            {
                FName ReferencerPackage = Referencer.PackageName;
                if (!New.Contains(ReferencerPackage) && !Modified.Contains(ReferencerPackage))
                {
                    Modified.Add(ReferencerPackage);
                    Recurse.Add(ReferencerPackage);
                }
            }
        }

        const TMap<FName, const FAssetPackageData*>& PackageMap = NewState.GetAssetPackageDataMap();
        for (FName const& Package: Modified)
        {
            const FAssetPackageData* Data     = PackageMap[Package];
            const FAssetPackageData* PrevData = OldState.GetAssetPackageData(Package);

            RecordEdit(Package, *Data, *PrevData);
        }
    }
    else
    {
        for (const TPair<FName, const FAssetPackageData*>& Pair: NewState.GetAssetPackageDataMap())
        {
            FName Name = Pair.Key;

            if (bEnginePackagesOnly && !Name.ToString().StartsWith(TEXT("/Engine/")))
            {
                continue;
            }

            if (!IsInRelevantChunk(NewState, Name))
            {
                continue;
            }

            const FAssetPackageData* Data     = Pair.Value;
            const FAssetPackageData* PrevData = OldState.GetAssetPackageData(Name);

            if (Data->DiskSize < 0)
            {
                newuncooked++;
            }

            newassets += NewState.GetAssetsByPackageName(Name).Num();

            if (!PrevData)
            {
                RecordAdd(Name, *Data);
                AssetPathFlags.FindOrAdd(Name) |= EAssetFlags::Add;
            }
            else if (Data->CookedHash != PrevData->CookedHash)
            {
                RecordEdit(Name, *Data, *PrevData);
                AssetPathFlags.FindOrAdd(Name) |= EAssetFlags::HashChange;
                PRAGMA_DISABLE_DEPRECATION_WARNINGS
                if (Data->PackageGuid != PrevData->PackageGuid)
                    PRAGMA_ENABLE_DEPRECATION_WARNINGS
                    {
                        AssetPathFlags.FindOrAdd(Name) |= EAssetFlags::GuidChange;
                    }
            }
            else
            {
                RecordNoChange(Name, *Data);
            }
            newtotal++;
        }
    }

    for (const TPair<FName, const FAssetPackageData*>& Pair: OldState.GetAssetPackageDataMap())
    {
        FName Name         = Pair.Key;

        FString NameString = Name.ToString();

        if (bEnginePackagesOnly && !NameString.StartsWith(TEXT("/Engine/")))
        {
            continue;
        }

        if (!IsInRelevantChunk(OldState, Name))
        {
            continue;
        }

        const FAssetPackageData* PrevData = Pair.Value;
        const FAssetPackageData* Data     = NewState.GetAssetPackageData(Name);

        if (PrevData->DiskSize < 0)
        {
            olduncooked++;
        }

        oldassets += OldState.GetAssetsByPackageName(Name).Num();

        if (!Data)
        {
            RecordDelete(Name, *PrevData);
            AssetPathFlags.FindOrAdd(Name) |= EAssetFlags::Remove;
        }
        oldtotal++;
    }

    // Propagate hash/guid changes down through referencers
    {
        TArray<FName> Recurse;
        AssetPathFlags.GetKeys(Recurse);

        for (int32 RecurseIndex = 0; RecurseIndex < Recurse.Num(); RecurseIndex++)
        {
            FName Package = Recurse[RecurseIndex];

            TArray<FAssetIdentifier> Referencers;

            // grab the hash/guid change flags, shift up to the dependency ones
            int32 PackageFlags = AssetPathFlags.FindOrAdd(Package);
            int32 NewFlags     = (PackageFlags & 0x0C) << 2;

            // If we have a dependency chain like C -> B -> A, and A changes, this does not
            // necessarily cause B's binary representation to change, but it can still impact C.
            // We must propagate the dependency change flags too, otherwise C will be marked as
            // non-deterministic when this happens.
            NewFlags |= PackageFlags & (EAssetFlags::DepGuidChange | EAssetFlags::DepHashChange);

            // don't bother touching anything if this asset or its dependencies didn't change
            if (NewFlags)
            {
                NewState.GetReferencers(Package, Referencers, UE::AssetRegistry::EDependencyCategory::Package, UE::AssetRegistry::EDependencyQuery::Hard);

                for (const FAssetIdentifier& Referencer: Referencers)
                {
                    FName RefPackage = Referencer.PackageName;

                    // merge new dependency flags in, add to the list if something changed
                    int32& RefFlags = AssetPathFlags.FindOrAdd(RefPackage);
                    int32 OldFlags  = RefFlags;
                    RefFlags |= NewFlags;
                    if (RefFlags != OldFlags)
                    {
                        Recurse.Add(RefPackage);
                    }
                }
            }
        }
    }

    FArchive* CSVFile = nullptr;

    if (bSaveCSV)
    {
        CSVFile = IFileManager::Get().CreateFileWriter(*CSVFilename);
        CSVFile->Logf(TEXT("old,%s"), *OldPath);
        CSVFile->Logf(TEXT("new,%s"), *NewPath);
        CSVFile->Logf(TEXT(""));
    }

    SummarizeDeterminism();
    LogChangedFiles(CSVFile, OldPath, NewPath);

    // start summary
    UE_LOG(LogDiffAssets, Display, TEXT("Summary:"));
    UE_LOG(LogDiffAssets, Display, TEXT("Old AssetRegistry: %s"), *OldPath);
    UE_LOG(LogDiffAssets, Display, TEXT("%d packages total, %d uncooked, %d cooked assets"), oldtotal, olduncooked, oldassets);
    UE_LOG(LogDiffAssets, Display, TEXT("New AssetRegistry: %s"), *NewPath);

    const float InvToMB = 1.0 / (1024 * 1024);

    // Overall Class Summary
    // Actually do the warnings in this run, since it's the overall one
    LogClassSummary(CSVFile, TEXT("Overall"), ChangeSummaryByClass, true, DeterminismByClass);

    // Chunk-by-chunk class summaries
    if (bGroupByChunk)
    {
        TArray<int32> ChunkIDs;
        ChangesByChunk.GetKeys(ChunkIDs);
        ChunkIDs.Sort();
        for (int32 ChunkID: ChunkIDs)
        {
            FString ChunkHeader = (ChunkID == -1) ? TEXT("Untagged") : FString::Printf(TEXT("Chunk %d"), ChunkID);
            if (ChangesByChunk[ChunkID].ChangesByClass.Num())
            {
                LogClassSummary(CSVFile, *ChunkHeader, ChangesByChunk[ChunkID].ChangesByClass, false, ChangesByChunk[ChunkID].Determinism);
            }
        }
    }

#if 0
	TArray<int32> Changelists;
	ChangeSummaryByChangelist.GetKeys(Changelists);
	
	Changelists.Sort([this](const int32& Lhs, const int32& Rhs) {
			const FChangeInfo& LHSChanges = ChangeSummaryByChangelist[Lhs];
			const FChangeInfo& RHSChanges = ChangeSummaryByChangelist[Rhs];
			return LHSChanges.GetTotalChangeSize() > RHSChanges.GetTotalChangeSize();
		});
	
	for (int32 Changelist : Changelists)
	{
		const FChangeInfo& Changes = ChangeSummaryByChangelist[Changelist];
		
		if (Changes.GetTotalChangeSize() == 0)
		{
			continue;
		}
		
		if (Changes.GetTotalChangeCount() < MinChangeCount || Changes.GetTotalChangeSize() < (MinChangeSizeMB*1024*1024))
		{
			continue;
		}
		
		if (Changelist)
		{
			UE_LOG(LogDiffAssets, Display, TEXT("%d: %.02f%% changes (%.02f MB Total)"),
				   Changelist, Changes.GetChangePercentage() * 100.0, Changes.GetTotalChangeSize() * InvToMB);
		}
		else
		{
			UE_LOG(LogDiffAssets, Display, TEXT("Unattributable (nondeterministic?): %.02f%% changes (%.02f MB Total)"),
				   Changes.GetChangePercentage() * 100.0, Changes.GetTotalChangeSize() * InvToMB);
		}
		
		if (Changes.Adds)
		{
			UE_LOG(LogDiffAssets, Display, TEXT("\t%d packages added,    %8.3f MB"), Changes.Adds, Changes.AddedBytes * InvToMB);
		}
		
		if (Changes.Changes)
		{
			UE_LOG(LogDiffAssets, Display, TEXT("\t%d packages modified, %8.3f MB"), Changes.Changes, Changes.ChangedBytes * InvToMB);
		}
		
		if (Changes.Deletes)
		{
			UE_LOG(LogDiffAssets, Display, TEXT("\t%d packages removed,  %8.3f MB"), Changes.Deletes, Changes.DeletedBytes * InvToMB);
		}
	}
#endif
    // these are parsed by scripts, so please don't modify :)
    UE_LOG(LogDiffAssets, Display, TEXT("%d total packages, %d uncooked, %d cooked assets"), newtotal, newuncooked, newassets);
    UE_LOG(LogDiffAssets, Display, TEXT("%d total unchanged,         %8.3f MB"), ChangeSummary.Unchanged, ChangeSummary.UnchangedBytes * InvToMB);
    UE_LOG(LogDiffAssets, Display, TEXT("%d total packages added,    %8.3f MB"), ChangeSummary.Adds, ChangeSummary.AddedBytes * InvToMB);
    UE_LOG(LogDiffAssets, Display, TEXT("%d total packages modified, %8.3f MB"), ChangeSummary.Changes, ChangeSummary.ChangedBytes * InvToMB);
    UE_LOG(LogDiffAssets, Display, TEXT("%d total packages removed,  %8.3f MB"), ChangeSummary.Deletes, ChangeSummary.DeletedBytes * InvToMB);

    UE_LOG(LogDiffAssets, Display, TEXT("Nondeterministic summary:"));
    UE_LOG(LogDiffAssets, Display, TEXT("direct   %d total packages modified, %8.3f MB"), NondeterministicSummary.Changes, NondeterministicSummary.ChangedBytes * InvToMB);
    UE_LOG(LogDiffAssets, Display, TEXT("indirect %d total packages modified, %8.3f MB"), IndirectNondeterministicSummary.Changes, IndirectNondeterministicSummary.ChangedBytes * InvToMB);

    // Warn when meeting or exceeding a certain total changed size, if that's enabled
    if (WarnTotalChangedSizeMB > 0 && ChangeSummary.ChangedBytes * InvToMB >= WarnTotalChangedSizeMB)
    {
        UE_LOG(LogDiffAssets, Warning, TEXT("Total Changed Bytes exceeded %d MB! (%8.3f MB)"), WarnTotalChangedSizeMB, ChangeSummary.ChangedBytes * InvToMB);
    }

    if (CSVFile)
    {
        CSVFile->Logf(TEXT(""));
        CSVFile->Logf(TEXT("Summary"));
        CSVFile->Logf(TEXT("total,%d"), newtotal);
        CSVFile->Logf(TEXT("unchanged,%lld,%lld"), ChangeSummary.Unchanged, ChangeSummary.UnchangedBytes);
        CSVFile->Logf(TEXT("added,%lld,%lld"), ChangeSummary.Adds, ChangeSummary.AddedBytes);
        CSVFile->Logf(TEXT("modified,%lld,%lld"), ChangeSummary.Changes, ChangeSummary.ChangedBytes);
        CSVFile->Logf(TEXT("removed,%lld,%lld"), ChangeSummary.Deletes, ChangeSummary.DeletedBytes);
        CSVFile->Logf(TEXT("direct non-deterministic,%lld,%lld"), NondeterministicSummary.Changes, NondeterministicSummary.ChangedBytes);
        CSVFile->Logf(TEXT("indirect non-deterministic,%lld,%lld"), IndirectNondeterministicSummary.Changes, IndirectNondeterministicSummary.ChangedBytes);
        delete CSVFile;
    }
}

bool UDiffAssetRegistriesCommandlet::LaunchP4(const FString& Args, TArray<FString>& Output, int32& OutReturnCode) const
{
    void* PipeRead  = nullptr;
    void* PipeWrite = nullptr;

    verify(FPlatformProcess::CreatePipe(PipeRead, PipeWrite));

    bool bInvoked = false;
    OutReturnCode = -1;
    FString StringOutput;
    FProcHandle ProcHandle = FPlatformProcess::CreateProc(TEXT("p4.exe"), *Args, false, true, true, nullptr, 0, nullptr, PipeWrite);
    if (ProcHandle.IsValid())
    {
        while (FPlatformProcess::IsProcRunning(ProcHandle))
        {
            FString ThisRead = FPlatformProcess::ReadPipe(PipeRead);
            StringOutput += ThisRead;
            //			Re-enable waits if constant pipe-querying is somehow damaging, but keep the wait SMALL
            //			if (ThisRead.Len() <= 0)
            //			{
            //				FPlatformProcess::Sleep(0.001f);
            //			}
        }

        StringOutput += FPlatformProcess::ReadPipe(PipeRead);
        FPlatformProcess::GetProcReturnCode(ProcHandle, &OutReturnCode);
        bInvoked = true;
    }
    else
    {
        UE_LOG(LogDiffAssets, Error, TEXT("Failed to launch p4."));
    }

    FPlatformProcess::ClosePipe(PipeRead, PipeWrite);

    StringOutput.ParseIntoArrayLines(Output);

    return bInvoked;
}