1635 lines
77 KiB
C++
1635 lines
77 KiB
C++
// Copyright Epic Games, Inc. All Rights Reserved.
|
|
|
|
/*=============================================================================
|
|
CookCommandlet.cpp: Commandlet for cooking content
|
|
=============================================================================*/
|
|
|
|
#include "Commandlets/CookCommandlet.h"
|
|
#include "HAL/PlatformFilemanager.h"
|
|
#include "HAL/PlatformApplicationMisc.h"
|
|
#include "Misc/MessageDialog.h"
|
|
#include "HAL/FileManager.h"
|
|
#include "Misc/CommandLine.h"
|
|
#include "Misc/FileHelper.h"
|
|
#include "Misc/Paths.h"
|
|
#include "Stats/StatsMisc.h"
|
|
#include "Misc/ConfigCacheIni.h"
|
|
#include "Misc/LocalTimestampDirectoryVisitor.h"
|
|
#include "Misc/App.h"
|
|
#include "Modules/ModuleManager.h"
|
|
#include "UObject/Class.h"
|
|
#include "UObject/UObjectIterator.h"
|
|
#include "UObject/Package.h"
|
|
#include "UObject/MetaData.h"
|
|
#include "Async/TaskGraphInterfaces.h"
|
|
#include "Misc/RedirectCollector.h"
|
|
#include "IPlatformFileSandboxWrapper.h"
|
|
#include "CookOnTheSide/CookOnTheFlyServer.h"
|
|
#include "Settings/ProjectPackagingSettings.h"
|
|
#include "EngineGlobals.h"
|
|
#include "Editor.h"
|
|
#include "Serialization/ArrayWriter.h"
|
|
#include "PackageHelperFunctions.h"
|
|
#include "GlobalShader.h"
|
|
#include "ShaderCompiler.h"
|
|
#include "Interfaces/ITargetPlatform.h"
|
|
#include "Interfaces/ITargetPlatformManagerModule.h"
|
|
#include "INetworkFileSystemModule.h"
|
|
#include "GameDelegates.h"
|
|
#include "CookerSettings.h"
|
|
#include "ShaderCompiler.h"
|
|
#include "HAL/MemoryMisc.h"
|
|
#include "ProfilingDebugging/CookStats.h"
|
|
#include "AssetRegistryModule.h"
|
|
#include "StudioAnalytics.h"
|
|
#include "Cooker/CookProfiling.h"
|
|
|
|
DEFINE_LOG_CATEGORY_STATIC(LogCookCommandlet, Log, All);
|
|
|
|
#if ENABLE_COOK_STATS
|
|
#include "ProfilingDebugging/ScopedTimers.h"
|
|
#include "AnalyticsEventAttribute.h"
|
|
#include "IAnalyticsProviderET.h"
|
|
#include "AnalyticsET.h"
|
|
|
|
namespace DetailedCookStats
|
|
{
|
|
FString CookProject;
|
|
FString TargetPlatforms;
|
|
double CookWallTimeSec = 0.0;
|
|
double StartupWallTimeSec = 0.0;
|
|
double CookByTheBookTimeSec = 0.0;
|
|
double StartCookByTheBookTimeSec = 0.0;
|
|
extern double TickCookOnTheSideTimeSec;
|
|
extern double TickCookOnTheSideLoadPackagesTimeSec;
|
|
extern double TickCookOnTheSideResolveRedirectorsTimeSec;
|
|
extern double TickCookOnTheSideSaveCookedPackageTimeSec;
|
|
extern double TickCookOnTheSideBeginPackageCacheForCookedPlatformDataTimeSec;
|
|
extern double TickCookOnTheSideFinishPackageCacheForCookedPlatformDataTimeSec;
|
|
extern double GameCookModificationDelegateTimeSec;
|
|
double TickLoopGCTimeSec = 0.0;
|
|
double TickLoopRecompileShaderRequestsTimeSec = 0.0;
|
|
double TickLoopShaderProcessAsyncResultsTimeSec = 0.0;
|
|
double TickLoopProcessDeferredCommandsTimeSec = 0.0;
|
|
double TickLoopTickCommandletStatsTimeSec = 0.0;
|
|
|
|
FCookStatsManager::FAutoRegisterCallback RegisterCookStats([](FCookStatsManager::AddStatFuncRef AddStat)
|
|
{
|
|
const FString StatName(TEXT("Cook.Profile"));
|
|
TArray<FCookStatsManager::StringKeyValue> Attrs;
|
|
#define ADD_COOK_STAT_FLT(Path, Name) AddStat(StatName, FCookStatsManager::CreateKeyValueArray(TEXT("Path"), TEXT(Path), TEXT(#Name), Name))
|
|
ADD_COOK_STAT_FLT(" 0", CookWallTimeSec);
|
|
ADD_COOK_STAT_FLT(" 0. 0", StartupWallTimeSec);
|
|
ADD_COOK_STAT_FLT(" 0. 1", CookByTheBookTimeSec);
|
|
ADD_COOK_STAT_FLT(" 0. 1. 0", StartCookByTheBookTimeSec);
|
|
ADD_COOK_STAT_FLT(" 0. 1. 0. 0", GameCookModificationDelegateTimeSec);
|
|
ADD_COOK_STAT_FLT(" 0. 1. 1", TickCookOnTheSideTimeSec);
|
|
ADD_COOK_STAT_FLT(" 0. 1. 1. 0", TickCookOnTheSideLoadPackagesTimeSec);
|
|
ADD_COOK_STAT_FLT(" 0. 1. 1. 1", TickCookOnTheSideSaveCookedPackageTimeSec);
|
|
ADD_COOK_STAT_FLT(" 0. 1. 1. 1. 0", TickCookOnTheSideResolveRedirectorsTimeSec);
|
|
ADD_COOK_STAT_FLT(" 0. 1. 1. 2", TickCookOnTheSideBeginPackageCacheForCookedPlatformDataTimeSec);
|
|
ADD_COOK_STAT_FLT(" 0. 1. 1. 3", TickCookOnTheSideFinishPackageCacheForCookedPlatformDataTimeSec);
|
|
ADD_COOK_STAT_FLT(" 0. 1. 2", TickLoopGCTimeSec);
|
|
ADD_COOK_STAT_FLT(" 0. 1. 3", TickLoopRecompileShaderRequestsTimeSec);
|
|
ADD_COOK_STAT_FLT(" 0. 1. 4", TickLoopShaderProcessAsyncResultsTimeSec);
|
|
ADD_COOK_STAT_FLT(" 0. 1. 5", TickLoopProcessDeferredCommandsTimeSec);
|
|
ADD_COOK_STAT_FLT(" 0. 1. 6", TickLoopTickCommandletStatsTimeSec);
|
|
#undef ADD_COOK_STAT_FLT
|
|
});
|
|
|
|
static void LogCookStats(const FString& CookCmdLine)
|
|
{
|
|
if (FStudioAnalytics::IsAvailable())
|
|
{
|
|
// convert filtered stats directly to an analytics event
|
|
TArray<FAnalyticsEventAttribute> StatAttrs;
|
|
|
|
// Sends each cook stat to the studio analytics system.
|
|
auto SendCookStatsToAnalytics = [&StatAttrs](const FString& StatName, const TArray<FCookStatsManager::StringKeyValue>& StatAttributes)
|
|
{
|
|
for (const auto& Attr: StatAttributes)
|
|
{
|
|
FString FormattedAttrName = StatName + "." + Attr.Key;
|
|
|
|
StatAttrs.Emplace(FormattedAttrName, Attr.Value);
|
|
}
|
|
};
|
|
|
|
// Now actually grab the stats
|
|
FCookStatsManager::LogCookStats(SendCookStatsToAnalytics);
|
|
|
|
// Record them all under cooking event
|
|
FStudioAnalytics::GetProvider().RecordEvent(TEXT("Core.Cooking"), StatAttrs);
|
|
|
|
FStudioAnalytics::GetProvider().BlockUntilFlushed(60.0f);
|
|
}
|
|
|
|
bool bSendCookAnalytics = false;
|
|
GConfig->GetBool(TEXT("CookAnalytics"), TEXT("SendAnalytics"), bSendCookAnalytics, GEngineIni);
|
|
|
|
if (GIsBuildMachine || FParse::Param(FCommandLine::Get(), TEXT("SendCookAnalytics")) || bSendCookAnalytics)
|
|
{
|
|
FString APIServerET;
|
|
|
|
if (GConfig->GetString(TEXT("CookAnalytics"), TEXT("APIServer"), APIServerET, GEngineIni))
|
|
{
|
|
FString AppId(TEXT("Cook"));
|
|
bool bUseLegacyCookProtocol = !GConfig->GetString(TEXT("CookAnalytics"), TEXT("AppId"), AppId, GEngineIni);
|
|
|
|
// Optionally create an analytics provider to send stats to for central collection.
|
|
TSharedPtr<IAnalyticsProviderET> CookAnalytics = FAnalyticsET::Get().CreateAnalyticsProvider(FAnalyticsET::Config(AppId, APIServerET, FString(), bUseLegacyCookProtocol));
|
|
if (CookAnalytics.IsValid())
|
|
{
|
|
CookAnalytics->SetUserID(FString::Printf(TEXT("%s\\%s"), FPlatformProcess::ComputerName(), FPlatformProcess::UserName(false)));
|
|
CookAnalytics->StartSession(MakeAnalyticsEventAttributeArray(
|
|
TEXT("Project"), CookProject,
|
|
TEXT("CmdLine"), CookCmdLine,
|
|
TEXT("IsBuildMachine"), GIsBuildMachine,
|
|
TEXT("TargetPlatforms"), TargetPlatforms));
|
|
|
|
TArray<FString> CookStatsToSend;
|
|
const bool bUseWhitelist = GConfig->GetArray(TEXT("CookAnalytics"), TEXT("CookStats"), CookStatsToSend, GEngineIni) > 0;
|
|
// Sends each cook stat to the analytics provider.
|
|
auto SendCookStatsToAnalytics = [CookAnalytics, &CookStatsToSend, bUseWhitelist](const FString& StatName, const TArray<FCookStatsManager::StringKeyValue>& StatAttributes)
|
|
{
|
|
if (!bUseWhitelist || CookStatsToSend.Contains(StatName))
|
|
{
|
|
// convert filtered stats directly to an analytics event
|
|
TArray<FAnalyticsEventAttribute> StatAttrs;
|
|
StatAttrs.Reset(StatAttributes.Num());
|
|
for (const auto& Attr: StatAttributes)
|
|
{
|
|
StatAttrs.Emplace(Attr.Key, Attr.Value);
|
|
}
|
|
CookAnalytics->RecordEvent(StatName, StatAttrs);
|
|
}
|
|
else
|
|
{
|
|
UE_LOG(LogCookCommandlet, Verbose, TEXT("[%s] not present on cook analytics whitelist"), *StatName);
|
|
}
|
|
};
|
|
|
|
FCookStatsManager::LogCookStats(SendCookStatsToAnalytics);
|
|
}
|
|
}
|
|
}
|
|
|
|
/** Used for custom logging of DDC Resource usage stats. */
|
|
struct FDDCResourceUsageStat
|
|
{
|
|
public:
|
|
FDDCResourceUsageStat(FString InAssetType, double InTotalTimeSec, bool bIsGameThreadTime, double InSizeMB, int64 InAssetsBuilt): AssetType(MoveTemp(InAssetType)), TotalTimeSec(InTotalTimeSec), GameThreadTimeSec(bIsGameThreadTime ? InTotalTimeSec : 0.0), SizeMB(InSizeMB), AssetsBuilt(InAssetsBuilt) {}
|
|
void Accumulate(const FDDCResourceUsageStat& OtherStat)
|
|
{
|
|
TotalTimeSec += OtherStat.TotalTimeSec;
|
|
GameThreadTimeSec += OtherStat.GameThreadTimeSec;
|
|
SizeMB += OtherStat.SizeMB;
|
|
AssetsBuilt += OtherStat.AssetsBuilt;
|
|
}
|
|
FString AssetType;
|
|
double TotalTimeSec;
|
|
double GameThreadTimeSec;
|
|
double SizeMB;
|
|
int64 AssetsBuilt;
|
|
};
|
|
|
|
/** Used for custom TSet comparison of DDC Resource usage stats. */
|
|
struct FDDCResourceUsageStatKeyFuncs: BaseKeyFuncs<FDDCResourceUsageStat, FString, false>
|
|
{
|
|
static const FString& GetSetKey(const FDDCResourceUsageStat& Element) { return Element.AssetType; }
|
|
static bool Matches(const FString& A, const FString& B) { return A == B; }
|
|
static uint32 GetKeyHash(const FString& Key) { return GetTypeHash(Key); }
|
|
};
|
|
|
|
/** Used to store profile data for custom logging. */
|
|
struct FCookProfileData
|
|
{
|
|
public:
|
|
FCookProfileData(FString InPath, FString InKey, FString InValue): Path(MoveTemp(InPath)), Key(MoveTemp(InKey)), Value(MoveTemp(InValue)) {}
|
|
FString Path;
|
|
FString Key;
|
|
FString Value;
|
|
};
|
|
|
|
// instead of printing the usage stats generically, we capture them so we can log a subset of them in an easy-to-read way.
|
|
TSet<FDDCResourceUsageStat, FDDCResourceUsageStatKeyFuncs> DDCResourceUsageStats;
|
|
TArray<FCookStatsManager::StringKeyValue> DDCSummaryStats;
|
|
TArray<FCookProfileData> CookProfileData;
|
|
TArray<FString> StatCategories;
|
|
TMap<FString, TArray<FCookStatsManager::StringKeyValue>> StatsInCategories;
|
|
|
|
/** this functor will take a collected cooker stat and log it out using some custom formatting based on known stats that are collected.. */
|
|
auto LogStatsFunc = [&DDCResourceUsageStats, &DDCSummaryStats, &CookProfileData, &StatCategories, &StatsInCategories](const FString& StatName, const TArray<FCookStatsManager::StringKeyValue>& StatAttributes)
|
|
{
|
|
// Some stats will use custom formatting to make a visibly pleasing summary.
|
|
bool bStatUsedCustomFormatting = false;
|
|
|
|
if (StatName == TEXT("DDC.Usage"))
|
|
{
|
|
// Don't even log this detailed DDC data. It's mostly only consumable by ingestion into pivot tools.
|
|
bStatUsedCustomFormatting = true;
|
|
}
|
|
else if (StatName.EndsWith(TEXT(".Usage"), ESearchCase::IgnoreCase))
|
|
{
|
|
// Anything that ends in .Usage is assumed to be an instance of FCookStats.FDDCResourceUsageStats. We'll log that using custom formatting.
|
|
FString AssetType = StatName;
|
|
AssetType.RemoveFromEnd(TEXT(".Usage"), ESearchCase::IgnoreCase);
|
|
// See if the asset has a subtype (found via the "Node" parameter")
|
|
const FCookStatsManager::StringKeyValue* AssetSubType = StatAttributes.FindByPredicate([](const FCookStatsManager::StringKeyValue& Item)
|
|
{
|
|
return Item.Key == TEXT("Node");
|
|
});
|
|
if (AssetSubType && AssetSubType->Value.Len() > 0)
|
|
{
|
|
AssetType += FString::Printf(TEXT(" (%s)"), *AssetSubType->Value);
|
|
}
|
|
// Pull the Time and Size attributes and AddOrAccumulate them into the set of stats. Ugly string/container manipulation code courtesy of UE4/C++.
|
|
const FCookStatsManager::StringKeyValue* AssetTimeSecAttr = StatAttributes.FindByPredicate([](const FCookStatsManager::StringKeyValue& Item)
|
|
{
|
|
return Item.Key == TEXT("TimeSec");
|
|
});
|
|
double AssetTimeSec = 0.0;
|
|
if (AssetTimeSecAttr)
|
|
{
|
|
LexFromString(AssetTimeSec, *AssetTimeSecAttr->Value);
|
|
}
|
|
const FCookStatsManager::StringKeyValue* AssetSizeMBAttr = StatAttributes.FindByPredicate([](const FCookStatsManager::StringKeyValue& Item)
|
|
{
|
|
return Item.Key == TEXT("MB");
|
|
});
|
|
double AssetSizeMB = 0.0;
|
|
if (AssetSizeMBAttr)
|
|
{
|
|
LexFromString(AssetSizeMB, *AssetSizeMBAttr->Value);
|
|
}
|
|
const FCookStatsManager::StringKeyValue* ThreadNameAttr = StatAttributes.FindByPredicate([](const FCookStatsManager::StringKeyValue& Item)
|
|
{
|
|
return Item.Key == TEXT("ThreadName");
|
|
});
|
|
bool bIsGameThreadTime = ThreadNameAttr != nullptr && ThreadNameAttr->Value == TEXT("GameThread");
|
|
|
|
const FCookStatsManager::StringKeyValue* HitOrMissAttr = StatAttributes.FindByPredicate([](const FCookStatsManager::StringKeyValue& Item)
|
|
{
|
|
return Item.Key == TEXT("HitOrMiss");
|
|
});
|
|
bool bWasMiss = HitOrMissAttr != nullptr && HitOrMissAttr->Value == TEXT("Miss");
|
|
int64 AssetsBuilt = 0;
|
|
if (bWasMiss)
|
|
{
|
|
const FCookStatsManager::StringKeyValue* CountAttr = StatAttributes.FindByPredicate([](const FCookStatsManager::StringKeyValue& Item)
|
|
{
|
|
return Item.Key == TEXT("Count");
|
|
});
|
|
if (CountAttr)
|
|
{
|
|
LexFromString(AssetsBuilt, *CountAttr->Value);
|
|
}
|
|
}
|
|
|
|
FDDCResourceUsageStat Stat(AssetType, AssetTimeSec, bIsGameThreadTime, AssetSizeMB, AssetsBuilt);
|
|
FDDCResourceUsageStat* ExistingStat = DDCResourceUsageStats.Find(Stat.AssetType);
|
|
if (ExistingStat)
|
|
{
|
|
ExistingStat->Accumulate(Stat);
|
|
}
|
|
else
|
|
{
|
|
DDCResourceUsageStats.Add(Stat);
|
|
}
|
|
bStatUsedCustomFormatting = true;
|
|
}
|
|
else if (StatName == TEXT("DDC.Summary"))
|
|
{
|
|
DDCSummaryStats.Append(StatAttributes);
|
|
bStatUsedCustomFormatting = true;
|
|
}
|
|
else if (StatName == TEXT("Cook.Profile"))
|
|
{
|
|
if (StatAttributes.Num() >= 2)
|
|
{
|
|
CookProfileData.Emplace(StatAttributes[0].Value, StatAttributes[1].Key, StatAttributes[1].Value);
|
|
}
|
|
bStatUsedCustomFormatting = true;
|
|
}
|
|
|
|
// if a stat doesn't use custom formatting, just spit out the raw info.
|
|
if (!bStatUsedCustomFormatting)
|
|
{
|
|
TArray<FCookStatsManager::StringKeyValue>& StatsInCategory = StatsInCategories.FindOrAdd(StatName);
|
|
if (StatsInCategory.Num() == 0)
|
|
{
|
|
StatCategories.Add(StatName);
|
|
}
|
|
StatsInCategory.Append(StatAttributes);
|
|
}
|
|
};
|
|
|
|
FCookStatsManager::LogCookStats(LogStatsFunc);
|
|
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("Misc Cook Stats"));
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("==============="));
|
|
for (FString& StatCategory: StatCategories)
|
|
{
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("%s"), *StatCategory);
|
|
TArray<FCookStatsManager::StringKeyValue>& StatsInCategory = StatsInCategories.FindOrAdd(StatCategory);
|
|
|
|
// log each key/value pair, with the equal signs lined up.
|
|
for (const FCookStatsManager::StringKeyValue& StatKeyValue: StatsInCategory)
|
|
{
|
|
UE_LOG(LogCookCommandlet, Display, TEXT(" %s=%s"), *StatKeyValue.Key, *StatKeyValue.Value);
|
|
}
|
|
}
|
|
|
|
// DDC Usage stats are custom formatted, and the above code just accumulated them into a TSet. Now log it with our special formatting for readability.
|
|
if (CookProfileData.Num() > 0)
|
|
{
|
|
UE_LOG(LogCookCommandlet, Display, TEXT(""));
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("Cook Profile"));
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("============"));
|
|
for (const auto& ProfileEntry: CookProfileData)
|
|
{
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("%s.%s=%s"), *ProfileEntry.Path, *ProfileEntry.Key, *ProfileEntry.Value);
|
|
}
|
|
}
|
|
if (DDCSummaryStats.Num() > 0)
|
|
{
|
|
UE_LOG(LogCookCommandlet, Display, TEXT(""));
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("DDC Summary Stats"));
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("================="));
|
|
for (const auto& Attr: DDCSummaryStats)
|
|
{
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("%-16s=%10s"), *Attr.Key, *Attr.Value);
|
|
}
|
|
}
|
|
if (DDCResourceUsageStats.Num() > 0)
|
|
{
|
|
// sort the list
|
|
TArray<FDDCResourceUsageStat> SortedDDCResourceUsageStats;
|
|
SortedDDCResourceUsageStats.Empty(DDCResourceUsageStats.Num());
|
|
for (const FDDCResourceUsageStat& Stat: DDCResourceUsageStats)
|
|
{
|
|
SortedDDCResourceUsageStats.Emplace(Stat);
|
|
}
|
|
SortedDDCResourceUsageStats.Sort([](const FDDCResourceUsageStat& LHS, const FDDCResourceUsageStat& RHS)
|
|
{
|
|
return LHS.TotalTimeSec > RHS.TotalTimeSec;
|
|
});
|
|
|
|
UE_LOG(LogCookCommandlet, Display, TEXT(""));
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("DDC Resource Stats"));
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("======================================================================================================="));
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("Asset Type Total Time (Sec) GameThread Time (Sec) Assets Built MB Processed"));
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("---------------------------------- ---------------- --------------------- ------------ ------------"));
|
|
for (const FDDCResourceUsageStat& Stat: SortedDDCResourceUsageStats)
|
|
{
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("%-34s %16.2f %21.2f %12d %12.2f"), *Stat.AssetType, Stat.TotalTimeSec, Stat.GameThreadTimeSec, Stat.AssetsBuilt, Stat.SizeMB);
|
|
}
|
|
}
|
|
}
|
|
} // namespace DetailedCookStats
|
|
#endif
|
|
|
|
UCookCommandlet::UCookCommandlet(const FObjectInitializer& ObjectInitializer)
|
|
: Super(ObjectInitializer)
|
|
{
|
|
|
|
LogToConsole = false;
|
|
}
|
|
|
|
bool UCookCommandlet::CookOnTheFly(FGuid InstanceId, int32 Timeout, bool bForceClose, const TArray<ITargetPlatform*>& TargetPlatforms)
|
|
{
|
|
UCookOnTheFlyServer* CookOnTheFlyServer = NewObject<UCookOnTheFlyServer>();
|
|
|
|
struct FScopeRootObject
|
|
{
|
|
UObject* Object;
|
|
FScopeRootObject(UObject* InObject): Object(InObject)
|
|
{
|
|
Object->AddToRoot();
|
|
}
|
|
|
|
~FScopeRootObject()
|
|
{
|
|
Object->RemoveFromRoot();
|
|
}
|
|
};
|
|
|
|
// make sure that the cookonthefly server doesn't get cleaned up while we are garbage collecting below :)
|
|
FScopeRootObject S(CookOnTheFlyServer);
|
|
|
|
UCookerSettings const* CookerSettings = GetDefault<UCookerSettings>();
|
|
ECookInitializationFlags IterateFlags = ECookInitializationFlags::Iterative;
|
|
|
|
ECookInitializationFlags CookFlags = ECookInitializationFlags::None;
|
|
CookFlags |= bIterativeCooking ? IterateFlags : ECookInitializationFlags::None;
|
|
CookFlags |= bSkipEditorContent ? ECookInitializationFlags::SkipEditorContent : ECookInitializationFlags::None;
|
|
CookFlags |= bUnversioned ? ECookInitializationFlags::Unversioned : ECookInitializationFlags::None;
|
|
CookOnTheFlyServer->Initialize(ECookMode::CookOnTheFly, CookFlags);
|
|
|
|
bool BindAnyPort = InstanceId.IsValid();
|
|
|
|
if (CookOnTheFlyServer->StartNetworkFileServer(BindAnyPort, TargetPlatforms) == false)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (InstanceId.IsValid())
|
|
{
|
|
if (CookOnTheFlyServer->BroadcastFileserverPresence(InstanceId) == false)
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
if (bNoShaderCooking)
|
|
{
|
|
GShaderCompilingManager->SkipShaderCompilation(true);
|
|
}
|
|
|
|
// Garbage collection should happen when either
|
|
// 1. We have cooked a map (configurable asset type)
|
|
// 2. We have cooked non-map packages and...
|
|
// a. we have accumulated 50 (configurable) of these since the last GC.
|
|
// b. we have been idle for 20 (configurable) seconds.
|
|
struct FCookOnTheFlyGCController
|
|
{
|
|
public:
|
|
FCookOnTheFlyGCController(const UCookOnTheFlyServer* COtFServer)
|
|
: PackagesPerGC(COtFServer->GetPackagesPerGC()), IdleTimeToGC(COtFServer->GetIdleTimeToGC()), bShouldGC(true), PackagesCookedSinceLastGC(0), LastCookActionTime(FPlatformTime::Seconds())
|
|
{}
|
|
|
|
/** Tntended to be called with stats from a UCookOnTheFlyServer::TickCookOnTheSide() call. Determines if we should be calling GC after TickCookOnTheSide(). */
|
|
void Update(uint32 CookedCount, UCookOnTheFlyServer::ECookOnTheSideResult ResultFlags)
|
|
{
|
|
if (ResultFlags & (UCookOnTheFlyServer::COSR_CookedMap | UCookOnTheFlyServer::COSR_CookedPackage | UCookOnTheFlyServer::COSR_WaitingOnCache))
|
|
{
|
|
LastCookActionTime = FPlatformTime::Seconds();
|
|
}
|
|
|
|
if (ResultFlags & UCookOnTheFlyServer::COSR_RequiresGC)
|
|
{
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("Cooker cooked a map since last gc... collecting garbage"));
|
|
bShouldGC |= true;
|
|
}
|
|
|
|
PackagesCookedSinceLastGC += CookedCount;
|
|
if ((PackagesPerGC > 0) && (PackagesCookedSinceLastGC > PackagesPerGC))
|
|
{
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("Cooker has exceeded max number of non map packages since last gc"));
|
|
bShouldGC |= true;
|
|
}
|
|
|
|
// we don't want to gc if we are waiting on cache of objects. this could clean up objects which we will need to reload next frame
|
|
bPosteponeGC = (ResultFlags & UCookOnTheFlyServer::COSR_WaitingOnCache) != 0;
|
|
}
|
|
|
|
/** Runs GC if Update() determined it should happen. Also checks the idle time against the limit, and runs GC then if packages have been loaded. */
|
|
void ConditionallyCollectGarbage(const UCookOnTheFlyServer* COtFServer)
|
|
{
|
|
if (!bShouldGC)
|
|
{
|
|
if (PackagesCookedSinceLastGC > 0 && IdleTimeToGC > 0)
|
|
{
|
|
double IdleTime = FPlatformTime::Seconds() - LastCookActionTime;
|
|
if (IdleTime >= IdleTimeToGC)
|
|
{
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("Cooker has been idle for long time gc"));
|
|
bShouldGC |= true;
|
|
}
|
|
}
|
|
|
|
if (!bShouldGC && COtFServer->HasExceededMaxMemory())
|
|
{
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("Cooker has exceeded max memory usage collecting garbage"));
|
|
bShouldGC |= true;
|
|
}
|
|
}
|
|
|
|
if (bShouldGC && !bPosteponeGC)
|
|
{
|
|
Reset();
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("GC..."));
|
|
CollectGarbage(RF_NoFlags);
|
|
}
|
|
}
|
|
|
|
private:
|
|
/** Resets counters and flags used to determine when we should GC. */
|
|
void Reset()
|
|
{
|
|
bShouldGC = false;
|
|
PackagesCookedSinceLastGC = 0;
|
|
}
|
|
|
|
private:
|
|
const uint32 PackagesPerGC;
|
|
const double IdleTimeToGC;
|
|
|
|
bool bShouldGC;
|
|
uint32 PackagesCookedSinceLastGC;
|
|
double LastCookActionTime;
|
|
bool bPosteponeGC;
|
|
|
|
} CookOnTheFlyGCController(CookOnTheFlyServer);
|
|
|
|
FDateTime LastConnectionTime = FDateTime::UtcNow();
|
|
bool bHadConnection = false;
|
|
|
|
while (!IsEngineExitRequested())
|
|
{
|
|
uint32 CookedPkgCount = 0;
|
|
uint32 TickResults = CookOnTheFlyServer->TickCookOnTheSide(/*TimeSlice =*/10.f, CookedPkgCount, ShowProgress ? ECookTickFlags::None : ECookTickFlags::HideProgressDisplay);
|
|
|
|
// Flush the asset registry before GC
|
|
FAssetRegistryModule::TickAssetRegistry(-1.0f);
|
|
|
|
CookOnTheFlyGCController.Update(CookedPkgCount, (UCookOnTheFlyServer::ECookOnTheSideResult)TickResults);
|
|
CookOnTheFlyGCController.ConditionallyCollectGarbage(CookOnTheFlyServer);
|
|
|
|
// force at least a tick shader compilation even if we are requesting stuff
|
|
CookOnTheFlyServer->TickRecompileShaderRequests();
|
|
GShaderCompilingManager->ProcessAsyncResults(true, false);
|
|
|
|
while ((CookOnTheFlyServer->HasRemainingWork() == false) && !IsEngineExitRequested())
|
|
{
|
|
CookOnTheFlyServer->TickRecompileShaderRequests();
|
|
|
|
// Shaders need to be updated
|
|
GShaderCompilingManager->ProcessAsyncResults(true, false);
|
|
|
|
ProcessDeferredCommands();
|
|
|
|
// handle server timeout
|
|
if (InstanceId.IsValid() || bForceClose)
|
|
{
|
|
if (CookOnTheFlyServer->NumConnections() > 0)
|
|
{
|
|
bHadConnection = true;
|
|
LastConnectionTime = FDateTime::UtcNow();
|
|
}
|
|
|
|
if ((FDateTime::UtcNow() - LastConnectionTime) > FTimespan::FromSeconds(Timeout))
|
|
{
|
|
uint32 Result = FMessageDialog::Open(EAppMsgType::YesNo, NSLOCTEXT("UnrealEd", "FileServerIdle", "The file server did not receive any connections in the past 3 minutes. Would you like to shut it down?"));
|
|
|
|
if (Result == EAppReturnType::No && !bForceClose)
|
|
{
|
|
LastConnectionTime = FDateTime::UtcNow();
|
|
}
|
|
else
|
|
{
|
|
RequestEngineExit(TEXT("Cook file server idle"));
|
|
}
|
|
}
|
|
else if (bHadConnection && (CookOnTheFlyServer->NumConnections() == 0) && bForceClose) // immediately shut down if we previously had a connection and now do not
|
|
{
|
|
RequestEngineExit(TEXT("Cook file server lost last connection"));
|
|
}
|
|
}
|
|
|
|
CookOnTheFlyGCController.ConditionallyCollectGarbage(CookOnTheFlyServer);
|
|
|
|
CookOnTheFlyServer->WaitForRequests(100 /* timeoutMs */);
|
|
}
|
|
}
|
|
|
|
if (bNoShaderCooking)
|
|
{
|
|
GShaderCompilingManager->SkipShaderCompilation(false);
|
|
}
|
|
|
|
CookOnTheFlyServer->EndNetworkFileServer();
|
|
return true;
|
|
}
|
|
|
|
/* UCommandlet interface
|
|
*****************************************************************************/
|
|
|
|
int32 UCookCommandlet::Main(const FString& CmdLineParams)
|
|
{
|
|
COOK_STAT(double CookStartTime = FPlatformTime::Seconds());
|
|
Params = CmdLineParams;
|
|
ParseCommandLine(*Params, Tokens, Switches);
|
|
|
|
bCookOnTheFly = Switches.Contains(TEXT("COOKONTHEFLY")); // Prototype cook-on-the-fly server
|
|
bCookAll = Switches.Contains(TEXT("COOKALL")); // Cook everything
|
|
bUnversioned = Switches.Contains(TEXT("UNVERSIONED")); // Save all cooked packages without versions. These are then assumed to be current version on load. This is dangerous but results in smaller patch sizes.
|
|
bGenerateStreamingInstallManifests = Switches.Contains(TEXT("MANIFESTS")); // Generate manifests for building streaming install packages
|
|
bIterativeCooking = Switches.Contains(TEXT("ITERATE"));
|
|
bSkipEditorContent = Switches.Contains(TEXT("SKIPEDITORCONTENT")); // This won't save out any packages in Engine/Content/Editor*
|
|
bErrorOnEngineContentUse = Switches.Contains(TEXT("ERRORONENGINECONTENTUSE"));
|
|
bUseSerializationForGeneratingPackageDependencies = Switches.Contains(TEXT("UseSerializationForGeneratingPackageDependencies"));
|
|
bCookSinglePackage = Switches.Contains(TEXT("cooksinglepackagenorefs"));
|
|
bKeepSinglePackageRefs = Switches.Contains(TEXT("cooksinglepackage")); // This is a legacy parameter; it's a minor misnomer since singlepackage implies norefs, but we want to avoiding changing the behavior
|
|
bCookSinglePackage = bCookSinglePackage || bKeepSinglePackageRefs;
|
|
bVerboseCookerWarnings = Switches.Contains(TEXT("verbosecookerwarnings"));
|
|
bPartialGC = Switches.Contains(TEXT("Partialgc"));
|
|
ShowErrorCount = !Switches.Contains(TEXT("DIFFONLY"));
|
|
ShowProgress = !Switches.Contains(TEXT("DIFFONLY"));
|
|
bNoShaderCooking = bCookOnTheFly; // Do not cook any shaders into the shader maps. Always true if we are running w/ cook on the fly
|
|
|
|
COOK_STAT(DetailedCookStats::CookProject = FApp::GetProjectName());
|
|
|
|
ITargetPlatformManagerModule& TPM = GetTargetPlatformManagerRef();
|
|
if (bCookOnTheFly)
|
|
{
|
|
// In cook on the fly, if the user did not provide a targetplatform on the commandline, then we do not intialize any platforms up front; we wait for the first connection.
|
|
// TPM.GetActiveTargetPlatforms defaults to the currently running platform (e.g. Windows, with editor) in the no-target case, so we need to only call GetActiveTargetPlatforms
|
|
// if targetplatform was on the commandline
|
|
FString Unused;
|
|
TArray<ITargetPlatform*> TargetPlatforms;
|
|
if (FParse::Value(FCommandLine::Get(), TEXT("TARGETPLATFORM="), Unused))
|
|
{
|
|
TargetPlatforms = TPM.GetActiveTargetPlatforms();
|
|
}
|
|
|
|
// parse instance identifier
|
|
FString InstanceIdString;
|
|
bool bForceClose = Switches.Contains(TEXT("FORCECLOSE"));
|
|
|
|
FGuid InstanceId;
|
|
if (FParse::Value(*Params, TEXT("InstanceId="), InstanceIdString))
|
|
{
|
|
if (!FGuid::Parse(InstanceIdString, InstanceId))
|
|
{
|
|
UE_LOG(LogCookCommandlet, Warning, TEXT("Invalid InstanceId on command line: %s"), *InstanceIdString);
|
|
}
|
|
}
|
|
|
|
int32 Timeout = 180;
|
|
if (!FParse::Value(*Params, TEXT("timeout="), Timeout))
|
|
{
|
|
Timeout = 180;
|
|
}
|
|
|
|
CookOnTheFly(InstanceId, Timeout, bForceClose, TargetPlatforms);
|
|
}
|
|
else if (Switches.Contains(TEXT("COOKWORKER")))
|
|
{
|
|
CookAsCookWorker();
|
|
}
|
|
else
|
|
{
|
|
const TArray<ITargetPlatform*>& Platforms = TPM.GetActiveTargetPlatforms();
|
|
bool bCookMP = Switches.Contains(TEXT("CookMP"));
|
|
if (bCookMP)
|
|
{
|
|
CookByTheBookList(Platforms);
|
|
}
|
|
else
|
|
{
|
|
CookByTheBook(Platforms);
|
|
}
|
|
|
|
if (GShaderCompilerStats)
|
|
{
|
|
GShaderCompilerStats->WriteStats();
|
|
}
|
|
|
|
// Use -LogCookStats to log the results to the command line after the cook (happens automatically on a build machine)
|
|
COOK_STAT(
|
|
{
|
|
double Now = FPlatformTime::Seconds();
|
|
DetailedCookStats::CookWallTimeSec = Now - GStartTime;
|
|
DetailedCookStats::StartupWallTimeSec = CookStartTime - GStartTime;
|
|
DetailedCookStats::LogCookStats(CmdLineParams);
|
|
});
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
bool UCookCommandlet::CookByTheBook(const TArray<ITargetPlatform*>& Platforms)
|
|
{
|
|
TRACE_CPUPROFILER_EVENT_SCOPE_ON_CHANNEL(CookByTheBook, CookChannel);
|
|
|
|
COOK_STAT(FScopedDurationTimer CookByTheBookTimer(DetailedCookStats::CookByTheBookTimeSec));
|
|
UCookOnTheFlyServer* CookOnTheFlyServer = NewObject<UCookOnTheFlyServer>();
|
|
|
|
struct FScopeRootObject
|
|
{
|
|
UObject* Object;
|
|
FScopeRootObject(UObject* InObject): Object(InObject)
|
|
{
|
|
Object->AddToRoot();
|
|
}
|
|
|
|
~FScopeRootObject()
|
|
{
|
|
Object->RemoveFromRoot();
|
|
}
|
|
};
|
|
|
|
// make sure that the cookonthefly server doesn't get cleaned up while we are garbage collecting below :)
|
|
FScopeRootObject S(CookOnTheFlyServer);
|
|
|
|
UCookerSettings const* CookerSettings = GetDefault<UCookerSettings>();
|
|
ECookInitializationFlags IterateFlags = ECookInitializationFlags::Iterative;
|
|
|
|
if (Switches.Contains(TEXT("IterateSharedCookedbuild")))
|
|
{
|
|
// Add shared build flag to method flag, and enable iterative
|
|
IterateFlags |= ECookInitializationFlags::IterateSharedBuild;
|
|
|
|
bIterativeCooking = true;
|
|
}
|
|
|
|
ECookInitializationFlags CookFlags = ECookInitializationFlags::IncludeServerMaps;
|
|
CookFlags |= bIterativeCooking ? IterateFlags : ECookInitializationFlags::None;
|
|
CookFlags |= bSkipEditorContent ? ECookInitializationFlags::SkipEditorContent : ECookInitializationFlags::None;
|
|
CookFlags |= bUseSerializationForGeneratingPackageDependencies ? ECookInitializationFlags::UseSerializationForPackageDependencies : ECookInitializationFlags::None;
|
|
CookFlags |= bUnversioned ? ECookInitializationFlags::Unversioned : ECookInitializationFlags::None;
|
|
CookFlags |= bVerboseCookerWarnings ? ECookInitializationFlags::OutputVerboseCookerWarnings : ECookInitializationFlags::None;
|
|
CookFlags |= bPartialGC ? ECookInitializationFlags::EnablePartialGC : ECookInitializationFlags::None;
|
|
bool bTestCook = Switches.Contains(TEXT("TestCook"));
|
|
CookFlags |= bTestCook ? ECookInitializationFlags::TestCook : ECookInitializationFlags::None;
|
|
CookFlags |= Switches.Contains(TEXT("LogDebugInfo")) ? ECookInitializationFlags::LogDebugInfo : ECookInitializationFlags::None;
|
|
CookFlags |= Switches.Contains(TEXT("IgnoreIniSettingsOutOfDate")) || CookerSettings->bIgnoreIniSettingsOutOfDateForIteration ? ECookInitializationFlags::IgnoreIniSettingsOutOfDate : ECookInitializationFlags::None;
|
|
CookFlags |= Switches.Contains(TEXT("IgnoreScriptPackagesOutOfDate")) || CookerSettings->bIgnoreScriptPackagesOutOfDateForIteration ? ECookInitializationFlags::IgnoreScriptPackagesOutOfDate : ECookInitializationFlags::None;
|
|
|
|
//////////////////////////////////////////////////////////////////////////
|
|
// parse commandline options
|
|
|
|
FString DLCName;
|
|
FParse::Value(*Params, TEXT("DLCNAME="), DLCName);
|
|
|
|
FString ChildCookFile;
|
|
FParse::Value(*Params, TEXT("cookchild="), ChildCookFile);
|
|
|
|
FString BasedOnReleaseVersion;
|
|
FParse::Value(*Params, TEXT("BasedOnReleaseVersion="), BasedOnReleaseVersion);
|
|
|
|
FString CreateReleaseVersion;
|
|
FParse::Value(*Params, TEXT("CreateReleaseVersion="), CreateReleaseVersion);
|
|
|
|
FString OutputDirectoryOverride;
|
|
FParse::Value(*Params, TEXT("OutputDir="), OutputDirectoryOverride);
|
|
|
|
TArray<FString> CmdLineMapEntries;
|
|
TArray<FString> CmdLineDirEntries;
|
|
TArray<FString> CmdLineCultEntries;
|
|
TArray<FString> CmdLineNeverCookDirEntries;
|
|
for (int32 SwitchIdx = 0; SwitchIdx < Switches.Num(); SwitchIdx++)
|
|
{
|
|
const FString& Switch = Switches[SwitchIdx];
|
|
|
|
auto GetSwitchValueElements = [&Switch](const FString SwitchKey) -> TArray<FString>
|
|
{
|
|
TArray<FString> ValueElements;
|
|
if (Switch.StartsWith(SwitchKey + TEXT("=")) == true)
|
|
{
|
|
FString ValuesList = Switch.Right(Switch.Len() - (SwitchKey + TEXT("=")).Len());
|
|
|
|
// Allow support for -KEY=Value1+Value2+Value3 as well as -KEY=Value1 -KEY=Value2
|
|
for (int32 PlusIdx = ValuesList.Find(TEXT("+"), ESearchCase::CaseSensitive); PlusIdx != INDEX_NONE; PlusIdx = ValuesList.Find(TEXT("+"), ESearchCase::CaseSensitive))
|
|
{
|
|
const FString ValueElement = ValuesList.Left(PlusIdx);
|
|
ValueElements.Add(ValueElement);
|
|
|
|
ValuesList.RightInline(ValuesList.Len() - (PlusIdx + 1), false);
|
|
}
|
|
ValueElements.Add(ValuesList);
|
|
}
|
|
return ValueElements;
|
|
};
|
|
|
|
// Check for -MAP=<name of map> entries
|
|
CmdLineMapEntries += GetSwitchValueElements(TEXT("MAP"));
|
|
|
|
// Check for -COOKDIR=<path to directory> entries
|
|
const FString CookDirPrefix = TEXT("COOKDIR=");
|
|
if (Switch.StartsWith(CookDirPrefix))
|
|
{
|
|
FString Entry = Switch.Mid(CookDirPrefix.Len()).TrimQuotes();
|
|
FPaths::NormalizeDirectoryName(Entry);
|
|
CmdLineDirEntries.Add(Entry);
|
|
}
|
|
|
|
// Check for -NEVERCOOKDIR=<path to directory> entries
|
|
for (FString& NeverCookDir: GetSwitchValueElements(TEXT("NEVERCOOKDIR")))
|
|
{
|
|
FPaths::NormalizeDirectoryName(NeverCookDir);
|
|
CmdLineNeverCookDirEntries.Add(MoveTemp(NeverCookDir));
|
|
}
|
|
|
|
// Check for -COOKCULTURES=<culture name> entries
|
|
CmdLineCultEntries += GetSwitchValueElements(TEXT("COOKCULTURES"));
|
|
}
|
|
|
|
CookOnTheFlyServer->Initialize(ECookMode::CookByTheBook, CookFlags, OutputDirectoryOverride);
|
|
|
|
// Add any map sections specified on command line
|
|
TArray<FString> AlwaysCookMapList;
|
|
|
|
// Add the default map section
|
|
// GEditor->LoadMapListFromIni(TEXT("AlwaysCookMaps"), AlwaysCookMapList);
|
|
|
|
TArray<FString> MapList;
|
|
// Add any map sections specified on command line
|
|
/*GEditor->ParseMapSectionIni(*Params, MapList);
|
|
|
|
if (MapList.Num() == 0 && !bCookSinglePackage)
|
|
{
|
|
// If we didn't find any maps look in the project settings for maps
|
|
|
|
UProjectPackagingSettings* PackagingSettings = Cast<UProjectPackagingSettings>(UProjectPackagingSettings::StaticClass()->GetDefaultObject());
|
|
|
|
for (const auto& MapToCook : PackagingSettings->MapsToCook)
|
|
{
|
|
MapList.Add(MapToCook.FilePath);
|
|
}
|
|
}*/
|
|
|
|
// Add any map specified on the command line.
|
|
for (const auto& MapName: CmdLineMapEntries)
|
|
{
|
|
MapList.Add(MapName);
|
|
}
|
|
|
|
TArray<FString> MapIniSections;
|
|
FString SectionStr;
|
|
if (FParse::Value(*Params, TEXT("MAPINISECTION="), SectionStr))
|
|
{
|
|
if (SectionStr.Contains(TEXT("+")))
|
|
{
|
|
TArray<FString> Sections;
|
|
SectionStr.ParseIntoArray(Sections, TEXT("+"), true);
|
|
for (int32 Index = 0; Index < Sections.Num(); Index++)
|
|
{
|
|
MapIniSections.Add(Sections[Index]);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
MapIniSections.Add(SectionStr);
|
|
}
|
|
}
|
|
|
|
if (!bCookSinglePackage)
|
|
{
|
|
// Put the always cook map list at the front of the map list
|
|
AlwaysCookMapList.Append(MapList);
|
|
Swap(MapList, AlwaysCookMapList);
|
|
}
|
|
|
|
// Set the list of cultures to cook as those on the commandline, if specified.
|
|
// Otherwise, use the project packaging settings.
|
|
TArray<FString> CookCultures;
|
|
if (Switches.ContainsByPredicate([](const FString& Switch) -> bool
|
|
{
|
|
return Switch.StartsWith("COOKCULTURES=");
|
|
}))
|
|
{
|
|
CookCultures = CmdLineCultEntries;
|
|
}
|
|
else
|
|
{
|
|
UProjectPackagingSettings* const PackagingSettings = Cast<UProjectPackagingSettings>(UProjectPackagingSettings::StaticClass()->GetDefaultObject());
|
|
CookCultures = PackagingSettings->CulturesToStage;
|
|
}
|
|
|
|
//////////////////////////////////////////////////////////////////////////
|
|
// start cook by the book
|
|
ECookByTheBookOptions CookOptions = ECookByTheBookOptions::None;
|
|
|
|
CookOptions |= bCookAll ? ECookByTheBookOptions::CookAll : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("MAPSONLY")) ? ECookByTheBookOptions::MapsOnly : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("NODEV")) ? ECookByTheBookOptions::NoDevContent : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("FullLoadAndSave")) ? ECookByTheBookOptions::FullLoadAndSave : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("PackageStore")) ? ECookByTheBookOptions::PackageStore : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("NoGameAlwaysCook")) ? ECookByTheBookOptions::NoGameAlwaysCookPackages : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("DisableUnsolicitedPackages")) ? (ECookByTheBookOptions::SkipHardReferences | ECookByTheBookOptions::SkipSoftReferences) : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("NoDefaultMaps")) ? ECookByTheBookOptions::NoDefaultMaps : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("SkipSoftReferences")) ? ECookByTheBookOptions::SkipSoftReferences : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("SkipHardReferences")) ? ECookByTheBookOptions::SkipHardReferences : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("CookAgainstFixedBase")) ? ECookByTheBookOptions::CookAgainstFixedBase : ECookByTheBookOptions::None;
|
|
CookOptions |= (Switches.Contains(TEXT("DlcLoadMainAssetRegistry")) || !bErrorOnEngineContentUse) ? ECookByTheBookOptions::DlcLoadMainAssetRegistry : ECookByTheBookOptions::None;
|
|
|
|
if (bCookSinglePackage)
|
|
{
|
|
const ECookByTheBookOptions SinglePackageFlags = ECookByTheBookOptions::NoAlwaysCookMaps | ECookByTheBookOptions::NoDefaultMaps | ECookByTheBookOptions::NoGameAlwaysCookPackages |
|
|
ECookByTheBookOptions::NoInputPackages | ECookByTheBookOptions::NoSlatePackages | ECookByTheBookOptions::SkipSoftReferences | ECookByTheBookOptions::ForceDisableSaveGlobalShaders;
|
|
CookOptions |= SinglePackageFlags;
|
|
CookOptions |= bKeepSinglePackageRefs ? ECookByTheBookOptions::None : ECookByTheBookOptions::SkipHardReferences;
|
|
}
|
|
|
|
// Also append any cookdirs from the project ini files; these dirs are relative to the game content directory or start with a / root
|
|
if (!(CookOptions & ECookByTheBookOptions::NoGameAlwaysCookPackages))
|
|
{
|
|
const UProjectPackagingSettings* const PackagingSettings = GetDefault<UProjectPackagingSettings>();
|
|
for (const FDirectoryPath& DirToCook: PackagingSettings->DirectoriesToAlwaysCook)
|
|
{
|
|
FString LocalPath;
|
|
if (FPackageName::TryConvertGameRelativePackagePathToLocalPath(DirToCook.Path, LocalPath))
|
|
{
|
|
CmdLineDirEntries.Add(LocalPath);
|
|
}
|
|
else
|
|
{
|
|
UE_LOG(LogCook, Warning, TEXT("'ProjectSettings -> PackagingSettings -> Directories to always cook' has invalid element '%s'"), *DirToCook.Path);
|
|
}
|
|
}
|
|
}
|
|
|
|
UCookOnTheFlyServer::FCookByTheBookStartupOptions StartupOptions;
|
|
|
|
// Validate target platforms and add them to StartupOptions
|
|
for (ITargetPlatform* TargetPlatform: Platforms)
|
|
{
|
|
if (TargetPlatform)
|
|
{
|
|
if (TargetPlatform->HasEditorOnlyData())
|
|
{
|
|
UE_LOG(LogCook, Warning, TEXT("Target platform \"%s\" is an editor platform and can not be a cook target"), *TargetPlatform->PlatformName());
|
|
}
|
|
else
|
|
{
|
|
StartupOptions.TargetPlatforms.Add(TargetPlatform);
|
|
}
|
|
}
|
|
}
|
|
if (!StartupOptions.TargetPlatforms.Num())
|
|
{
|
|
UE_LOG(LogCook, Error, TEXT("No target platforms specified or all target platforms are invalid"));
|
|
return false;
|
|
}
|
|
|
|
Swap(StartupOptions.CookMaps, MapList);
|
|
Swap(StartupOptions.CookDirectories, CmdLineDirEntries);
|
|
Swap(StartupOptions.NeverCookDirectories, CmdLineNeverCookDirEntries);
|
|
Swap(StartupOptions.CookCultures, CookCultures);
|
|
Swap(StartupOptions.DLCName, DLCName);
|
|
Swap(StartupOptions.BasedOnReleaseVersion, BasedOnReleaseVersion);
|
|
Swap(StartupOptions.CreateReleaseVersion, CreateReleaseVersion);
|
|
Swap(StartupOptions.IniMapSections, MapIniSections);
|
|
StartupOptions.CookOptions = CookOptions;
|
|
StartupOptions.bErrorOnEngineContentUse = bErrorOnEngineContentUse;
|
|
StartupOptions.bGenerateDependenciesForMaps = Switches.Contains(TEXT("GenerateDependenciesForMaps"));
|
|
StartupOptions.bGenerateStreamingInstallManifests = bGenerateStreamingInstallManifests;
|
|
|
|
COOK_STAT(
|
|
{
|
|
for (const auto& Platform: Platforms)
|
|
{
|
|
DetailedCookStats::TargetPlatforms += Platform->PlatformName() + TEXT("+");
|
|
}
|
|
if (!DetailedCookStats::TargetPlatforms.IsEmpty())
|
|
{
|
|
DetailedCookStats::TargetPlatforms.RemoveFromEnd(TEXT("+"));
|
|
}
|
|
});
|
|
|
|
do
|
|
{
|
|
{
|
|
COOK_STAT(FScopedDurationTimer StartCookByTheBookTimer(DetailedCookStats::StartCookByTheBookTimeSec));
|
|
CookOnTheFlyServer->StartCookByTheBook(StartupOptions);
|
|
}
|
|
|
|
// Garbage collection should happen when either
|
|
// 1. We have cooked a map (configurable asset type)
|
|
// 2. We have cooked non-map packages and...
|
|
// a. we have accumulated 50 (configurable) of these since the last GC.
|
|
// b. we have been idle for 20 (configurable) seconds.
|
|
bool bShouldGC = false;
|
|
FString GCReason;
|
|
|
|
// megamoth
|
|
uint32 NonMapPackageCountSinceLastGC = 0;
|
|
|
|
const uint32 PackagesPerGC = CookOnTheFlyServer->GetPackagesPerGC();
|
|
const double IdleTimeToGC = CookOnTheFlyServer->GetIdleTimeToGC();
|
|
const uint32 PackagesPerPartialGC = CookOnTheFlyServer->GetPackagesPerPartialGC();
|
|
|
|
double LastCookActionTime = FPlatformTime::Seconds();
|
|
|
|
FDateTime LastConnectionTime = FDateTime::UtcNow();
|
|
bool bHadConnection = false;
|
|
|
|
while (CookOnTheFlyServer->IsCookByTheBookRunning())
|
|
{
|
|
DECLARE_SCOPE_CYCLE_COUNTER(TEXT("CookByTheBook.MainLoop"), STAT_CookByTheBook_MainLoop, STATGROUP_LoadTime);
|
|
{
|
|
uint32 TickResults = 0;
|
|
static const float CookOnTheSideTimeSlice = 10.0f;
|
|
|
|
TickResults = CookOnTheFlyServer->TickCookOnTheSide(CookOnTheSideTimeSlice, NonMapPackageCountSinceLastGC, ShowProgress ? ECookTickFlags::None : ECookTickFlags::HideProgressDisplay);
|
|
|
|
{
|
|
UE_SCOPED_COOKTIMER_AND_DURATION(CookByTheBook_ShaderProcessAsync, DetailedCookStats::TickLoopShaderProcessAsyncResultsTimeSec);
|
|
GShaderCompilingManager->ProcessAsyncResults(true, false);
|
|
}
|
|
|
|
// Flush the asset registry before GC
|
|
{
|
|
UE_SCOPED_COOKTIMER(CookByTheBook_TickAssetRegistry);
|
|
FAssetRegistryModule::TickAssetRegistry(-1.0f);
|
|
}
|
|
|
|
auto DumpMemStats = []()
|
|
{
|
|
FGenericMemoryStats MemStats;
|
|
GMalloc->GetAllocatorStats(MemStats);
|
|
for (const auto& Item: MemStats.Data)
|
|
{
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("Item %s = %d"), *Item.Key, Item.Value);
|
|
}
|
|
};
|
|
|
|
const bool bHasExceededMaxMemory = CookOnTheFlyServer->HasExceededMaxMemory();
|
|
// We should GC if we have packages to collect and we've been idle for some time.
|
|
const bool bExceededPackagesPerGC = (PackagesPerGC > 0) && (NonMapPackageCountSinceLastGC > PackagesPerGC);
|
|
const bool bWaitingOnObjectCache = ((TickResults & UCookOnTheFlyServer::COSR_WaitingOnCache) == 0);
|
|
|
|
if (!bWaitingOnObjectCache && bExceededPackagesPerGC) // if we are waiting on things to cache then ignore the exceeded packages per gc
|
|
{
|
|
bShouldGC = true;
|
|
GCReason = TEXT("Exceeded packages per GC");
|
|
}
|
|
else if (bHasExceededMaxMemory) // if we are exceeding memory then we need to gc (this can cause thrashing if the cooker loads the same stuff into memory next tick
|
|
{
|
|
bShouldGC = true;
|
|
GCReason = TEXT("Exceeded Max Memory");
|
|
|
|
int32 JobsToLogAt = GShaderCompilingManager->GetNumRemainingJobs();
|
|
|
|
UE_SCOPED_COOKTIMER(CookByTheBook_ShaderJobFlush);
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("Detected max mem exceeded - forcing shader compilation flush"));
|
|
while (true)
|
|
{
|
|
int32 NumRemainingJobs = GShaderCompilingManager->GetNumRemainingJobs();
|
|
if (NumRemainingJobs < 1000)
|
|
{
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("Finished flushing shader jobs at %d"), NumRemainingJobs);
|
|
break;
|
|
}
|
|
|
|
if (NumRemainingJobs < JobsToLogAt)
|
|
{
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("Flushing shader jobs, remaining jobs %d"), NumRemainingJobs);
|
|
}
|
|
|
|
GShaderCompilingManager->ProcessAsyncResults(false, false);
|
|
|
|
FPlatformProcess::Sleep(0.05);
|
|
|
|
// GShaderCompilingManager->FinishAllCompilation();
|
|
}
|
|
}
|
|
else if ((TickResults & UCookOnTheFlyServer::COSR_RequiresGC) != 0) // cooker loaded some object which needs to be cleaned up before the cooker can proceed so force gc
|
|
{
|
|
GCReason = TEXT("COSR_RequiresGC");
|
|
bShouldGC = true;
|
|
}
|
|
|
|
bShouldGC |= bTestCook; // testing cooking / gc path
|
|
|
|
if (bShouldGC)
|
|
{
|
|
UE_SCOPED_COOKTIMER_AND_DURATION(CookByTheBook_GC, DetailedCookStats::TickLoopGCTimeSec);
|
|
bShouldGC = false;
|
|
|
|
int32 NumObjectsBeforeGC = GUObjectArray.GetObjectArrayNumMinusAvailable();
|
|
int32 NumObjectsAvailableBeforeGC = GUObjectArray.GetObjectArrayEstimatedAvailable();
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("GarbageCollection...%s (%s)"), (bPartialGC ? TEXT(" partial gc") : TEXT("")), *GCReason);
|
|
GCReason = FString();
|
|
|
|
DumpMemStats();
|
|
|
|
CollectGarbage(RF_NoFlags);
|
|
|
|
int32 NumObjectsAfterGC = GUObjectArray.GetObjectArrayNumMinusAvailable();
|
|
int32 NumObjectsAvailableAfterGC = GUObjectArray.GetObjectArrayEstimatedAvailable();
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("%s GC before %d available %d after %d available %d"), (bPartialGC ? TEXT("Partial") : TEXT("Full")), NumObjectsBeforeGC, NumObjectsAvailableBeforeGC, NumObjectsAfterGC, NumObjectsAvailableAfterGC);
|
|
|
|
DumpMemStats();
|
|
|
|
NonMapPackageCountSinceLastGC = 0;
|
|
}
|
|
|
|
{
|
|
UE_SCOPED_COOKTIMER_AND_DURATION(CookByTheBook_RecompileShaderRequests, DetailedCookStats::TickLoopRecompileShaderRequestsTimeSec);
|
|
CookOnTheFlyServer->TickRecompileShaderRequests();
|
|
|
|
FPlatformProcess::Sleep(0.0f);
|
|
}
|
|
|
|
{
|
|
UE_SCOPED_COOKTIMER_AND_DURATION(CookByTheBook_ProcessDeferredCommands, DetailedCookStats::TickLoopProcessDeferredCommandsTimeSec);
|
|
ProcessDeferredCommands();
|
|
}
|
|
}
|
|
|
|
{
|
|
UE_SCOPED_COOKTIMER_AND_DURATION(CookByTheBook_TickCommandletStats, DetailedCookStats::TickLoopTickCommandletStatsTimeSec);
|
|
FStats::TickCommandletStats();
|
|
}
|
|
}
|
|
} while (bTestCook);
|
|
|
|
if (!bIterativeCooking && StartupOptions.DLCName.IsEmpty())
|
|
{
|
|
bool bFullReferencesExpected = !(CookOptions & ECookByTheBookOptions::SkipHardReferences);
|
|
VerifyEDLCookInfo(bFullReferencesExpected);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
void UCookCommandlet::ProcessDeferredCommands()
|
|
{
|
|
#if PLATFORM_MAC
|
|
// On Mac we need to process Cocoa events so that the console window for CookOnTheFlyServer is interactive
|
|
FPlatformApplicationMisc::PumpMessages(true);
|
|
#endif
|
|
|
|
// update task graph
|
|
FTaskGraphInterface::Get().ProcessThreadUntilIdle(ENamedThreads::GameThread);
|
|
|
|
// execute deferred commands
|
|
for (int32 DeferredCommandsIndex = 0; DeferredCommandsIndex < GEngine->DeferredCommands.Num(); ++DeferredCommandsIndex)
|
|
{
|
|
GEngine->Exec(GWorld, *GEngine->DeferredCommands[DeferredCommandsIndex], *GLog);
|
|
}
|
|
|
|
GEngine->DeferredCommands.Empty();
|
|
}
|
|
|
|
#pragma region MPCook
|
|
namespace UE
|
|
{
|
|
namespace Cook
|
|
{
|
|
struct FScopeRootObject
|
|
{
|
|
UObject* Object;
|
|
FScopeRootObject(UObject* InObject): Object(InObject)
|
|
{
|
|
Object->AddToRoot();
|
|
}
|
|
|
|
~FScopeRootObject()
|
|
{
|
|
Object->RemoveFromRoot();
|
|
}
|
|
};
|
|
} // namespace Cook
|
|
} // namespace UE
|
|
bool UCookCommandlet::CookByTheBookList(const TArray<ITargetPlatform*>& Platforms)
|
|
{
|
|
TRACE_CPUPROFILER_EVENT_SCOPE_ON_CHANNEL(CookByTheBook, CookChannel);
|
|
|
|
COOK_STAT(FScopedDurationTimer CookByTheBookTimer(DetailedCookStats::CookByTheBookTimeSec));
|
|
UCookOnTheFlyServer* CookOnTheFlyServer = NewObject<UCookOnTheFlyServer>();
|
|
|
|
// make sure that the cookonthefly server doesn't get cleaned up while we are garbage collecting below :)
|
|
UE::Cook::FScopeRootObject S(CookOnTheFlyServer);
|
|
|
|
UCookerSettings const* CookerSettings = GetDefault<UCookerSettings>();
|
|
ECookInitializationFlags IterateFlags = ECookInitializationFlags::Iterative;
|
|
|
|
if (Switches.Contains(TEXT("IterateSharedCookedbuild")))
|
|
{
|
|
// Add shared build flag to method flag, and enable iterative
|
|
IterateFlags |= ECookInitializationFlags::IterateSharedBuild;
|
|
|
|
bIterativeCooking = true;
|
|
}
|
|
|
|
ECookInitializationFlags CookFlags = ECookInitializationFlags::IncludeServerMaps;
|
|
CookFlags |= bIterativeCooking ? IterateFlags : ECookInitializationFlags::None;
|
|
CookFlags |= bSkipEditorContent ? ECookInitializationFlags::SkipEditorContent : ECookInitializationFlags::None;
|
|
CookFlags |= bUseSerializationForGeneratingPackageDependencies ? ECookInitializationFlags::UseSerializationForPackageDependencies : ECookInitializationFlags::None;
|
|
CookFlags |= bUnversioned ? ECookInitializationFlags::Unversioned : ECookInitializationFlags::None;
|
|
CookFlags |= bVerboseCookerWarnings ? ECookInitializationFlags::OutputVerboseCookerWarnings : ECookInitializationFlags::None;
|
|
CookFlags |= bPartialGC ? ECookInitializationFlags::EnablePartialGC : ECookInitializationFlags::None;
|
|
bool bTestCook = Switches.Contains(TEXT("TestCook"));
|
|
CookFlags |= bTestCook ? ECookInitializationFlags::TestCook : ECookInitializationFlags::None;
|
|
CookFlags |= Switches.Contains(TEXT("LogDebugInfo")) ? ECookInitializationFlags::LogDebugInfo : ECookInitializationFlags::None;
|
|
CookFlags |= Switches.Contains(TEXT("IgnoreIniSettingsOutOfDate")) || CookerSettings->bIgnoreIniSettingsOutOfDateForIteration ? ECookInitializationFlags::IgnoreIniSettingsOutOfDate : ECookInitializationFlags::None;
|
|
CookFlags |= Switches.Contains(TEXT("IgnoreScriptPackagesOutOfDate")) || CookerSettings->bIgnoreScriptPackagesOutOfDateForIteration ? ECookInitializationFlags::IgnoreScriptPackagesOutOfDate : ECookInitializationFlags::None;
|
|
|
|
//////////////////////////////////////////////////////////////////////////
|
|
// parse commandline options
|
|
|
|
FString DLCName;
|
|
FParse::Value(*Params, TEXT("DLCNAME="), DLCName);
|
|
|
|
FString ChildCookFile;
|
|
FParse::Value(*Params, TEXT("cookchild="), ChildCookFile);
|
|
|
|
FString BasedOnReleaseVersion;
|
|
FParse::Value(*Params, TEXT("BasedOnReleaseVersion="), BasedOnReleaseVersion);
|
|
|
|
FString CreateReleaseVersion;
|
|
FParse::Value(*Params, TEXT("CreateReleaseVersion="), CreateReleaseVersion);
|
|
|
|
FString OutputDirectoryOverride;
|
|
FParse::Value(*Params, TEXT("OutputDir="), OutputDirectoryOverride);
|
|
|
|
TArray<FString> CmdLineMapEntries;
|
|
TArray<FString> CmdLineDirEntries;
|
|
TArray<FString> CmdLineCultEntries;
|
|
TArray<FString> CmdLineNeverCookDirEntries;
|
|
for (int32 SwitchIdx = 0; SwitchIdx < Switches.Num(); SwitchIdx++)
|
|
{
|
|
const FString& Switch = Switches[SwitchIdx];
|
|
|
|
auto GetSwitchValueElements = [&Switch](const FString SwitchKey) -> TArray<FString>
|
|
{
|
|
TArray<FString> ValueElements;
|
|
if (Switch.StartsWith(SwitchKey + TEXT("=")) == true)
|
|
{
|
|
FString ValuesList = Switch.Right(Switch.Len() - (SwitchKey + TEXT("=")).Len());
|
|
|
|
// Allow support for -KEY=Value1+Value2+Value3 as well as -KEY=Value1 -KEY=Value2
|
|
for (int32 PlusIdx = ValuesList.Find(TEXT("+"), ESearchCase::CaseSensitive); PlusIdx != INDEX_NONE; PlusIdx = ValuesList.Find(TEXT("+"), ESearchCase::CaseSensitive))
|
|
{
|
|
const FString ValueElement = ValuesList.Left(PlusIdx);
|
|
ValueElements.Add(ValueElement);
|
|
|
|
ValuesList.RightInline(ValuesList.Len() - (PlusIdx + 1), false);
|
|
}
|
|
ValueElements.Add(ValuesList);
|
|
}
|
|
return ValueElements;
|
|
};
|
|
|
|
// Check for -MAP=<name of map> entries
|
|
CmdLineMapEntries += GetSwitchValueElements(TEXT("MAP"));
|
|
|
|
// Check for -COOKDIR=<path to directory> entries
|
|
const FString CookDirPrefix = TEXT("COOKDIR=");
|
|
if (Switch.StartsWith(CookDirPrefix))
|
|
{
|
|
FString Entry = Switch.Mid(CookDirPrefix.Len()).TrimQuotes();
|
|
FPaths::NormalizeDirectoryName(Entry);
|
|
CmdLineDirEntries.Add(Entry);
|
|
}
|
|
|
|
// Check for -NEVERCOOKDIR=<path to directory> entries
|
|
for (FString& NeverCookDir: GetSwitchValueElements(TEXT("NEVERCOOKDIR")))
|
|
{
|
|
FPaths::NormalizeDirectoryName(NeverCookDir);
|
|
CmdLineNeverCookDirEntries.Add(MoveTemp(NeverCookDir));
|
|
}
|
|
|
|
// Check for -COOKCULTURES=<culture name> entries
|
|
CmdLineCultEntries += GetSwitchValueElements(TEXT("COOKCULTURES"));
|
|
}
|
|
|
|
CookOnTheFlyServer->Initialize(ECookMode::CookByTheBook, CookFlags, OutputDirectoryOverride);
|
|
CookOnTheFlyServer->InitializeCookMP();
|
|
// Add any map sections specified on command line
|
|
TArray<FString> AlwaysCookMapList;
|
|
|
|
// Add the default map section
|
|
// GEditor->LoadMapListFromIni(TEXT("AlwaysCookMaps"), AlwaysCookMapList);
|
|
|
|
TArray<FString> MapList;
|
|
// Add any map sections specified on command line
|
|
/*GEditor->ParseMapSectionIni(*Params, MapList);
|
|
|
|
if (MapList.Num() == 0 && !bCookSinglePackage)
|
|
{
|
|
// If we didn't find any maps look in the project settings for maps
|
|
|
|
UProjectPackagingSettings* PackagingSettings = Cast<UProjectPackagingSettings>(UProjectPackagingSettings::StaticClass()->GetDefaultObject());
|
|
|
|
for (const auto& MapToCook : PackagingSettings->MapsToCook)
|
|
{
|
|
MapList.Add(MapToCook.FilePath);
|
|
}
|
|
}*/
|
|
|
|
// Add any map specified on the command line.
|
|
for (const auto& MapName: CmdLineMapEntries)
|
|
{
|
|
MapList.Add(MapName);
|
|
}
|
|
|
|
TArray<FString> MapIniSections;
|
|
FString SectionStr;
|
|
if (FParse::Value(*Params, TEXT("MAPINISECTION="), SectionStr))
|
|
{
|
|
if (SectionStr.Contains(TEXT("+")))
|
|
{
|
|
TArray<FString> Sections;
|
|
SectionStr.ParseIntoArray(Sections, TEXT("+"), true);
|
|
for (int32 Index = 0; Index < Sections.Num(); Index++)
|
|
{
|
|
MapIniSections.Add(Sections[Index]);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
MapIniSections.Add(SectionStr);
|
|
}
|
|
}
|
|
|
|
if (!bCookSinglePackage)
|
|
{
|
|
// Put the always cook map list at the front of the map list
|
|
AlwaysCookMapList.Append(MapList);
|
|
Swap(MapList, AlwaysCookMapList);
|
|
}
|
|
|
|
// Set the list of cultures to cook as those on the commandline, if specified.
|
|
// Otherwise, use the project packaging settings.
|
|
TArray<FString> CookCultures;
|
|
if (Switches.ContainsByPredicate([](const FString& Switch) -> bool
|
|
{
|
|
return Switch.StartsWith("COOKCULTURES=");
|
|
}))
|
|
{
|
|
CookCultures = CmdLineCultEntries;
|
|
}
|
|
else
|
|
{
|
|
UProjectPackagingSettings* const PackagingSettings = Cast<UProjectPackagingSettings>(UProjectPackagingSettings::StaticClass()->GetDefaultObject());
|
|
CookCultures = PackagingSettings->CulturesToStage;
|
|
}
|
|
|
|
//////////////////////////////////////////////////////////////////////////
|
|
// start cook by the book
|
|
ECookByTheBookOptions CookOptions = ECookByTheBookOptions::None;
|
|
|
|
CookOptions |= bCookAll ? ECookByTheBookOptions::CookAll : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("MAPSONLY")) ? ECookByTheBookOptions::MapsOnly : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("NODEV")) ? ECookByTheBookOptions::NoDevContent : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("FullLoadAndSave")) ? ECookByTheBookOptions::FullLoadAndSave : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("PackageStore")) ? ECookByTheBookOptions::PackageStore : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("NoGameAlwaysCook")) ? ECookByTheBookOptions::NoGameAlwaysCookPackages : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("DisableUnsolicitedPackages")) ? (ECookByTheBookOptions::SkipHardReferences | ECookByTheBookOptions::SkipSoftReferences) : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("NoDefaultMaps")) ? ECookByTheBookOptions::NoDefaultMaps : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("SkipSoftReferences")) ? ECookByTheBookOptions::SkipSoftReferences : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("SkipHardReferences")) ? ECookByTheBookOptions::SkipHardReferences : ECookByTheBookOptions::None;
|
|
CookOptions |= Switches.Contains(TEXT("CookAgainstFixedBase")) ? ECookByTheBookOptions::CookAgainstFixedBase : ECookByTheBookOptions::None;
|
|
CookOptions |= (Switches.Contains(TEXT("DlcLoadMainAssetRegistry")) || !bErrorOnEngineContentUse) ? ECookByTheBookOptions::DlcLoadMainAssetRegistry : ECookByTheBookOptions::None;
|
|
|
|
if (bCookSinglePackage)
|
|
{
|
|
const ECookByTheBookOptions SinglePackageFlags = ECookByTheBookOptions::NoAlwaysCookMaps | ECookByTheBookOptions::NoDefaultMaps | ECookByTheBookOptions::NoGameAlwaysCookPackages |
|
|
ECookByTheBookOptions::NoInputPackages | ECookByTheBookOptions::NoSlatePackages | ECookByTheBookOptions::SkipSoftReferences | ECookByTheBookOptions::ForceDisableSaveGlobalShaders;
|
|
CookOptions |= SinglePackageFlags;
|
|
CookOptions |= bKeepSinglePackageRefs ? ECookByTheBookOptions::None : ECookByTheBookOptions::SkipHardReferences;
|
|
}
|
|
|
|
// Also append any cookdirs from the project ini files; these dirs are relative to the game content directory or start with a / root
|
|
if (!(CookOptions & ECookByTheBookOptions::NoGameAlwaysCookPackages))
|
|
{
|
|
const UProjectPackagingSettings* const PackagingSettings = GetDefault<UProjectPackagingSettings>();
|
|
for (const FDirectoryPath& DirToCook: PackagingSettings->DirectoriesToAlwaysCook)
|
|
{
|
|
FString LocalPath;
|
|
if (FPackageName::TryConvertGameRelativePackagePathToLocalPath(DirToCook.Path, LocalPath))
|
|
{
|
|
CmdLineDirEntries.Add(LocalPath);
|
|
}
|
|
else
|
|
{
|
|
UE_LOG(LogCook, Warning, TEXT("'ProjectSettings -> PackagingSettings -> Directories to always cook' has invalid element '%s'"), *DirToCook.Path);
|
|
}
|
|
}
|
|
}
|
|
|
|
UCookOnTheFlyServer::FCookByTheBookStartupOptions StartupOptions;
|
|
|
|
// Validate target platforms and add them to StartupOptions
|
|
for (ITargetPlatform* TargetPlatform: Platforms)
|
|
{
|
|
if (TargetPlatform)
|
|
{
|
|
if (TargetPlatform->HasEditorOnlyData())
|
|
{
|
|
UE_LOG(LogCook, Warning, TEXT("Target platform \"%s\" is an editor platform and can not be a cook target"), *TargetPlatform->PlatformName());
|
|
}
|
|
else
|
|
{
|
|
StartupOptions.TargetPlatforms.Add(TargetPlatform);
|
|
}
|
|
}
|
|
}
|
|
if (!StartupOptions.TargetPlatforms.Num())
|
|
{
|
|
UE_LOG(LogCook, Error, TEXT("No target platforms specified or all target platforms are invalid"));
|
|
return false;
|
|
}
|
|
|
|
Swap(StartupOptions.CookMaps, MapList);
|
|
Swap(StartupOptions.CookDirectories, CmdLineDirEntries);
|
|
Swap(StartupOptions.NeverCookDirectories, CmdLineNeverCookDirEntries);
|
|
Swap(StartupOptions.CookCultures, CookCultures);
|
|
Swap(StartupOptions.DLCName, DLCName);
|
|
Swap(StartupOptions.BasedOnReleaseVersion, BasedOnReleaseVersion);
|
|
Swap(StartupOptions.CreateReleaseVersion, CreateReleaseVersion);
|
|
Swap(StartupOptions.IniMapSections, MapIniSections);
|
|
StartupOptions.CookOptions = CookOptions;
|
|
StartupOptions.bErrorOnEngineContentUse = bErrorOnEngineContentUse;
|
|
StartupOptions.bGenerateDependenciesForMaps = Switches.Contains(TEXT("GenerateDependenciesForMaps"));
|
|
StartupOptions.bGenerateStreamingInstallManifests = bGenerateStreamingInstallManifests;
|
|
|
|
COOK_STAT(
|
|
{
|
|
for (const auto& Platform: Platforms)
|
|
{
|
|
DetailedCookStats::TargetPlatforms += Platform->PlatformName() + TEXT("+");
|
|
}
|
|
if (!DetailedCookStats::TargetPlatforms.IsEmpty())
|
|
{
|
|
DetailedCookStats::TargetPlatforms.RemoveFromEnd(TEXT("+"));
|
|
}
|
|
});
|
|
|
|
do
|
|
{
|
|
{
|
|
COOK_STAT(FScopedDurationTimer StartCookByTheBookTimer(DetailedCookStats::StartCookByTheBookTimeSec));
|
|
CookOnTheFlyServer->StartCookByTheBook(StartupOptions);
|
|
CookOnTheFlyServer->StartCookMP(StartupOptions);
|
|
}
|
|
|
|
// Garbage collection should happen when either
|
|
// 1. We have cooked a map (configurable asset type)
|
|
// 2. We have cooked non-map packages and...
|
|
// a. we have accumulated 50 (configurable) of these since the last GC.
|
|
// b. we have been idle for 20 (configurable) seconds.
|
|
bool bShouldGC = false;
|
|
FString GCReason;
|
|
|
|
// megamoth
|
|
uint32 NonMapPackageCountSinceLastGC = 0;
|
|
|
|
const uint32 PackagesPerGC = CookOnTheFlyServer->GetPackagesPerGC();
|
|
const double IdleTimeToGC = CookOnTheFlyServer->GetIdleTimeToGC();
|
|
const uint32 PackagesPerPartialGC = CookOnTheFlyServer->GetPackagesPerPartialGC();
|
|
|
|
double LastCookActionTime = FPlatformTime::Seconds();
|
|
|
|
FDateTime LastConnectionTime = FDateTime::UtcNow();
|
|
bool bHadConnection = false;
|
|
|
|
while (CookOnTheFlyServer->IsCookByTheBookRunning())
|
|
{
|
|
DECLARE_SCOPE_CYCLE_COUNTER(TEXT("CookByTheBook.MainLoop"), STAT_CookByTheBook_MainLoop, STATGROUP_LoadTime);
|
|
{
|
|
uint32 TickResults = 0;
|
|
static const float CookOnTheSideTimeSlice = 10.0f;
|
|
|
|
TickResults = CookOnTheFlyServer->TickMainCookLoop(CookOnTheSideTimeSlice, NonMapPackageCountSinceLastGC, ShowProgress ? ECookTickFlags::None : ECookTickFlags::HideProgressDisplay);
|
|
|
|
{
|
|
UE_SCOPED_COOKTIMER_AND_DURATION(CookByTheBook_ShaderProcessAsync, DetailedCookStats::TickLoopShaderProcessAsyncResultsTimeSec);
|
|
GShaderCompilingManager->ProcessAsyncResults(true, false);
|
|
}
|
|
|
|
// Flush the asset registry before GC
|
|
{
|
|
UE_SCOPED_COOKTIMER(CookByTheBook_TickAssetRegistry);
|
|
FAssetRegistryModule::TickAssetRegistry(-1.0f);
|
|
}
|
|
|
|
auto DumpMemStats = []()
|
|
{
|
|
FGenericMemoryStats MemStats;
|
|
GMalloc->GetAllocatorStats(MemStats);
|
|
for (const auto& Item: MemStats.Data)
|
|
{
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("Item %s = %d"), *Item.Key, Item.Value);
|
|
}
|
|
};
|
|
|
|
const bool bHasExceededMaxMemory = CookOnTheFlyServer->HasExceededMaxMemory();
|
|
// We should GC if we have packages to collect and we've been idle for some time.
|
|
const bool bExceededPackagesPerGC = (PackagesPerGC > 0) && (NonMapPackageCountSinceLastGC > PackagesPerGC);
|
|
const bool bWaitingOnObjectCache = ((TickResults & UCookOnTheFlyServer::COSR_WaitingOnCache) == 0);
|
|
|
|
if (!bWaitingOnObjectCache && bExceededPackagesPerGC) // if we are waiting on things to cache then ignore the exceeded packages per gc
|
|
{
|
|
bShouldGC = true;
|
|
GCReason = TEXT("Exceeded packages per GC");
|
|
}
|
|
else if (bHasExceededMaxMemory) // if we are exceeding memory then we need to gc (this can cause thrashing if the cooker loads the same stuff into memory next tick
|
|
{
|
|
bShouldGC = true;
|
|
GCReason = TEXT("Exceeded Max Memory");
|
|
|
|
int32 JobsToLogAt = GShaderCompilingManager->GetNumRemainingJobs();
|
|
|
|
UE_SCOPED_COOKTIMER(CookByTheBook_ShaderJobFlush);
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("Detected max mem exceeded - forcing shader compilation flush"));
|
|
while (true)
|
|
{
|
|
int32 NumRemainingJobs = GShaderCompilingManager->GetNumRemainingJobs();
|
|
if (NumRemainingJobs < 1000)
|
|
{
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("Finished flushing shader jobs at %d"), NumRemainingJobs);
|
|
break;
|
|
}
|
|
|
|
if (NumRemainingJobs < JobsToLogAt)
|
|
{
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("Flushing shader jobs, remaining jobs %d"), NumRemainingJobs);
|
|
}
|
|
|
|
GShaderCompilingManager->ProcessAsyncResults(false, false);
|
|
|
|
FPlatformProcess::Sleep(0.05);
|
|
|
|
// GShaderCompilingManager->FinishAllCompilation();
|
|
}
|
|
}
|
|
else if ((TickResults & UCookOnTheFlyServer::COSR_RequiresGC) != 0) // cooker loaded some object which needs to be cleaned up before the cooker can proceed so force gc
|
|
{
|
|
GCReason = TEXT("COSR_RequiresGC");
|
|
bShouldGC = true;
|
|
}
|
|
|
|
bShouldGC |= bTestCook; // testing cooking / gc path
|
|
|
|
if (bShouldGC)
|
|
{
|
|
UE_SCOPED_COOKTIMER_AND_DURATION(CookByTheBook_GC, DetailedCookStats::TickLoopGCTimeSec);
|
|
bShouldGC = false;
|
|
|
|
int32 NumObjectsBeforeGC = GUObjectArray.GetObjectArrayNumMinusAvailable();
|
|
int32 NumObjectsAvailableBeforeGC = GUObjectArray.GetObjectArrayEstimatedAvailable();
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("GarbageCollection...%s (%s)"), (bPartialGC ? TEXT(" partial gc") : TEXT("")), *GCReason);
|
|
GCReason = FString();
|
|
|
|
DumpMemStats();
|
|
|
|
CollectGarbage(RF_NoFlags);
|
|
|
|
int32 NumObjectsAfterGC = GUObjectArray.GetObjectArrayNumMinusAvailable();
|
|
int32 NumObjectsAvailableAfterGC = GUObjectArray.GetObjectArrayEstimatedAvailable();
|
|
UE_LOG(LogCookCommandlet, Display, TEXT("%s GC before %d available %d after %d available %d"), (bPartialGC ? TEXT("Partial") : TEXT("Full")), NumObjectsBeforeGC, NumObjectsAvailableBeforeGC, NumObjectsAfterGC, NumObjectsAvailableAfterGC);
|
|
|
|
DumpMemStats();
|
|
|
|
NonMapPackageCountSinceLastGC = 0;
|
|
}
|
|
|
|
{
|
|
UE_SCOPED_COOKTIMER_AND_DURATION(CookByTheBook_RecompileShaderRequests, DetailedCookStats::TickLoopRecompileShaderRequestsTimeSec);
|
|
CookOnTheFlyServer->TickRecompileShaderRequests();
|
|
|
|
FPlatformProcess::Sleep(0.0f);
|
|
}
|
|
|
|
{
|
|
UE_SCOPED_COOKTIMER_AND_DURATION(CookByTheBook_ProcessDeferredCommands, DetailedCookStats::TickLoopProcessDeferredCommandsTimeSec);
|
|
ProcessDeferredCommands();
|
|
}
|
|
}
|
|
|
|
{
|
|
UE_SCOPED_COOKTIMER_AND_DURATION(CookByTheBook_TickCommandletStats, DetailedCookStats::TickLoopTickCommandletStatsTimeSec);
|
|
FStats::TickCommandletStats();
|
|
}
|
|
}
|
|
} while (bTestCook);
|
|
|
|
if (!bIterativeCooking && StartupOptions.DLCName.IsEmpty())
|
|
{
|
|
bool bFullReferencesExpected = !(CookOptions & ECookByTheBookOptions::SkipHardReferences);
|
|
VerifyEDLCookInfo(bFullReferencesExpected);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
bool UCookCommandlet::CookAsCookWorker()
|
|
{
|
|
#if OUTPUT_COOKTIMING
|
|
TRACE_CPUPROFILER_EVENT_SCOPE_ON_CHANNEL(CookAsCookWorker, CookChannel);
|
|
#endif // OUTPUT_COOKTIMING
|
|
|
|
UCookOnTheFlyServer* CookOnTheFlyServer = NewObject<UCookOnTheFlyServer>();
|
|
// make sure that the cookonthefly server doesn't get cleaned up while we are garbage collecting below
|
|
UE::Cook::FScopeRootObject S(CookOnTheFlyServer);
|
|
|
|
if (!CookOnTheFlyServer->TryInitializeCookWorker())
|
|
{
|
|
UE_LOG(LogCook, Display, TEXT("CookWorker initialization failed, aborting CookCommandlet."));
|
|
return false;
|
|
}
|
|
|
|
{
|
|
// megamoth
|
|
uint32 NonMapPackageCountSinceLastGC = 0;
|
|
DECLARE_SCOPE_CYCLE_COUNTER(TEXT("CookByTheBook.MainLoop"), STAT_CookByTheBook_MainLoop, STATGROUP_LoadTime);
|
|
while (CookOnTheFlyServer->IsInSession())
|
|
{
|
|
uint32 TickResults = 0;
|
|
static const float CookOnTheSideTimeSlice = 10.0f;
|
|
|
|
TickResults = CookOnTheFlyServer->TickMainCookLoop(CookOnTheSideTimeSlice, NonMapPackageCountSinceLastGC, ShowProgress ? ECookTickFlags::None : ECookTickFlags::HideProgressDisplay);
|
|
}
|
|
}
|
|
CookOnTheFlyServer->ShutdownCookAsCookWorker();
|
|
|
|
return true;
|
|
}
|
|
|
|
#pragma endregion |