// Copyright Epic Games, Inc. All Rights Reserved. #pragma once #include "CoreMinimal.h" #include "UObject/ObjectMacros.h" #include "Framework/Commands/Commands.h" #include "Commandlets/GatherTextCommandletBase.h" #include "Internationalization/StringTableCore.h" #include "GatherTextFromSourceCommandlet.generated.h" class Error; enum class EGatherTextSourceFileTypes : uint8 { None = 0, Cpp = 1 << 0, Ini = 1 << 1, }; ENUM_CLASS_FLAGS(EGatherTextSourceFileTypes); /** * UGatherTextFromSourceCommandlet: Localization commandlet that collects all text to be localized from the source code. */ UCLASS() class UGatherTextFromSourceCommandlet: public UGatherTextCommandletBase { GENERATED_UCLASS_BODY() private: #define LOC_DEFINE_REGION enum class EEditorOnlyDefineState : uint8 { Undefined, Defined, }; struct FSourceLocation { FSourceLocation() : File(), Line(INDEX_NONE) { } FSourceLocation(FString InFile, const int32 InLine) : File(MoveTemp(InFile)), Line(InLine) { } FString ToString() const { return (Line == INDEX_NONE) ? File : FString::Printf(TEXT("%s:%d"), *File, Line); } FString File; int32 Line; }; struct FParsedStringTableEntry { FString SourceString; FSourceLocation SourceLocation; FName PlatformName; bool bIsEditorOnly; }; struct FParsedStringTableEntryMetaData { FString MetaData; FSourceLocation SourceLocation; bool bIsEditorOnly; }; typedef TMap FParsedStringTableEntryMetaDataMap; struct FParsedStringTable { FString TableNamespace; FSourceLocation SourceLocation; TMap> TableEntries; TMap> MetaDataEntries; }; struct FSourceFileParseContext { bool AddManifestText(const FString& Token, const FString& Namespace, const FString& SourceText, const FManifestContext& Context); void PushMacroBlock(const FString& InBlockCtx); void PopMacroBlock(); void FlushMacroStack(); EEditorOnlyDefineState EvaluateEditorOnlyDefineState() const; void SetDefine(const FString& InDefineCtx); void RemoveDefine(const FString& InDefineCtx); void AddStringTable(const FName InTableId, const FString& InTableNamespace); void AddStringTableFromFile(const FName InTableId, const FString& InTableNamespace, const FString& InTableFilename, const FString& InRootPath); void AddStringTableEntry(const FName InTableId, const FString& InKey, const FString& InSourceString); void AddStringTableEntryMetaData(const FName InTableId, const FString& InKey, const FName InMetaDataId, const FString& InMetaData); // Working data EGatherTextSourceFileTypes FileTypes; FString Filename; int32 LineNumber; FName FilePlatformName; FString LineText; FString Namespace; FString RawStringLiteralClosingDelim; bool ExcludedRegion; bool EndParsingCurrentLine; bool WithinBlockComment; bool WithinLineComment; bool WithinStringLiteral; int32 WithinNamespaceDefineLineNumber; const TCHAR* WithinStartingLine; // Should editor-only data be included in this gather? bool ShouldGatherFromEditorOnlyData; // Discovered string table data from all files TMap ParsedStringTables; FSourceFileParseContext(UGatherTextFromSourceCommandlet* InOwnerCommandlet) : FileTypes(EGatherTextSourceFileTypes::None), Filename(), LineNumber(0), FilePlatformName(), LineText(), Namespace(), ExcludedRegion(false), EndParsingCurrentLine(false), WithinBlockComment(false), WithinLineComment(false), WithinStringLiteral(false), WithinNamespaceDefineLineNumber(INDEX_NONE), WithinStartingLine(nullptr), ShouldGatherFromEditorOnlyData(false), MacroBlockStack(), CachedEditorOnlyDefineState(), OwnerCommandlet(InOwnerCommandlet) { check(OwnerCommandlet); } private: bool AddStringTableImpl(const FName InTableId, const FString& InTableNamespace); bool AddStringTableEntryImpl(const FName InTableId, const FString& InKey, const FString& InSourceString, const FSourceLocation& InSourceLocation, const FName InPlatformName); bool AddStringTableEntryMetaDataImpl(const FName InTableId, const FString& InKey, const FName InMetaDataId, const FString& InMetaData, const FSourceLocation& InSourceLocation); // Working data TArray MacroBlockStack; mutable TOptional CachedEditorOnlyDefineState; UGatherTextFromSourceCommandlet* OwnerCommandlet; }; class FParsableDescriptor { public: virtual ~FParsableDescriptor() = default; virtual const FString& GetToken() const = 0; virtual void TryParse(const FString& Text, FSourceFileParseContext& Context) const = 0; bool MatchesFileTypes(const EGatherTextSourceFileTypes InFileTypes) { return EnumHasAnyFlags(ApplicableFileTypes, InFileTypes); } bool OverridesLongerTokens() { return bOverridesLongerTokens; } protected: EGatherTextSourceFileTypes ApplicableFileTypes = EGatherTextSourceFileTypes::None; bool bOverridesLongerTokens = false; }; class FPreProcessorDescriptor: public FParsableDescriptor { public: FPreProcessorDescriptor() { ApplicableFileTypes = EGatherTextSourceFileTypes::Cpp; bOverridesLongerTokens = true; } protected: static const FString DefineString; static const FString UndefString; static const FString IfString; static const FString IfDefString; static const FString ElIfString; static const FString ElseString; static const FString EndIfString; static const FString DefinedString; static const FString IniNamespaceString; }; class FDefineDescriptor: public FPreProcessorDescriptor { public: virtual const FString& GetToken() const override { return FPreProcessorDescriptor::DefineString; } virtual void TryParse(const FString& Text, FSourceFileParseContext& Context) const override; }; class FUndefDescriptor: public FPreProcessorDescriptor { public: virtual const FString& GetToken() const override { return FPreProcessorDescriptor::UndefString; } virtual void TryParse(const FString& Text, FSourceFileParseContext& Context) const override; }; class FIfDescriptor: public FPreProcessorDescriptor { public: virtual const FString& GetToken() const override { return FPreProcessorDescriptor::IfString; } virtual void TryParse(const FString& Text, FSourceFileParseContext& Context) const override; }; class FIfDefDescriptor: public FPreProcessorDescriptor { public: virtual const FString& GetToken() const override { return FPreProcessorDescriptor::IfDefString; } virtual void TryParse(const FString& Text, FSourceFileParseContext& Context) const override; }; class FElIfDescriptor: public FPreProcessorDescriptor { public: virtual const FString& GetToken() const override { return FPreProcessorDescriptor::ElIfString; } virtual void TryParse(const FString& Text, FSourceFileParseContext& Context) const override; }; class FElseDescriptor: public FPreProcessorDescriptor { public: virtual const FString& GetToken() const override { return FPreProcessorDescriptor::ElseString; } virtual void TryParse(const FString& Text, FSourceFileParseContext& Context) const override; }; class FEndIfDescriptor: public FPreProcessorDescriptor { public: virtual const FString& GetToken() const override { return FPreProcessorDescriptor::EndIfString; } virtual void TryParse(const FString& Text, FSourceFileParseContext& Context) const override; }; class FMacroDescriptor: public FParsableDescriptor { public: static const FString TextMacroString; FMacroDescriptor(FString InName) : Name(MoveTemp(InName)) { ApplicableFileTypes = EGatherTextSourceFileTypes::Cpp; } virtual const FString& GetToken() const override { return Name; } protected: bool ParseArgsFromMacro(const FString& Text, TArray& Args, FSourceFileParseContext& Context) const; static bool PrepareArgument(FString& Argument, bool IsAutoText, const FString& IdentForLogging, bool& OutHasQuotes); private: FString Name; }; class FUICommandMacroDescriptor: public FMacroDescriptor { public: FUICommandMacroDescriptor() : FMacroDescriptor(TEXT("UI_COMMAND")) { } virtual void TryParse(const FString& Text, FSourceFileParseContext& Context) const override; protected: FUICommandMacroDescriptor(FString InName) : FMacroDescriptor(MoveTemp(InName)) { } void TryParseArgs(const FString& Text, FSourceFileParseContext& Context, const TArray& Arguments, const int32 ArgIndexOffset) const; }; class FUICommandExtMacroDescriptor: public FUICommandMacroDescriptor { public: FUICommandExtMacroDescriptor() : FUICommandMacroDescriptor(TEXT("UI_COMMAND_EXT")) { } virtual void TryParse(const FString& Text, FSourceFileParseContext& Context) const override; }; class FStringMacroDescriptor: public FMacroDescriptor { public: enum EMacroArgSemantic { MAS_Namespace, MAS_Identifier, MAS_SourceText, }; struct FMacroArg { EMacroArgSemantic Semantic; bool IsAutoText; FMacroArg(EMacroArgSemantic InSema, bool InIsAutoText): Semantic(InSema), IsAutoText(InIsAutoText) {} }; FStringMacroDescriptor(FString InName, FMacroArg Arg0, FMacroArg Arg1, FMacroArg Arg2): FMacroDescriptor(InName) { ApplicableFileTypes = EGatherTextSourceFileTypes::Cpp | EGatherTextSourceFileTypes::Ini; Arguments.Add(Arg0); Arguments.Add(Arg1); Arguments.Add(Arg2); } FStringMacroDescriptor(FString InName, FMacroArg Arg0, FMacroArg Arg1): FMacroDescriptor(InName) { ApplicableFileTypes = EGatherTextSourceFileTypes::Cpp | EGatherTextSourceFileTypes::Ini; Arguments.Add(Arg0); Arguments.Add(Arg1); } FStringMacroDescriptor(FString InName, FMacroArg Arg0): FMacroDescriptor(InName) { ApplicableFileTypes = EGatherTextSourceFileTypes::Cpp | EGatherTextSourceFileTypes::Ini; Arguments.Add(Arg0); } virtual void TryParse(const FString& LineText, FSourceFileParseContext& Context) const override; private: TArray Arguments; }; class FStringTableMacroDescriptor: public FMacroDescriptor { public: FStringTableMacroDescriptor() : FMacroDescriptor(TEXT("LOCTABLE_NEW")) { } virtual void TryParse(const FString& Text, FSourceFileParseContext& Context) const override; }; class FStringTableFromFileMacroDescriptor: public FMacroDescriptor { public: FStringTableFromFileMacroDescriptor(FString InName, FString InRootPath): FMacroDescriptor(MoveTemp(InName)), RootPath(MoveTemp(InRootPath)) {} virtual void TryParse(const FString& Text, FSourceFileParseContext& Context) const override; private: FString RootPath; }; class FStringTableEntryMacroDescriptor: public FMacroDescriptor { public: FStringTableEntryMacroDescriptor() : FMacroDescriptor(TEXT("LOCTABLE_SETSTRING")) { } virtual void TryParse(const FString& Text, FSourceFileParseContext& Context) const override; }; class FStringTableEntryMetaDataMacroDescriptor: public FMacroDescriptor { public: FStringTableEntryMetaDataMacroDescriptor() : FMacroDescriptor(TEXT("LOCTABLE_SETMETA")) { } virtual void TryParse(const FString& Text, FSourceFileParseContext& Context) const override; }; class FIniNamespaceDescriptor: public FPreProcessorDescriptor { public: FIniNamespaceDescriptor() { ApplicableFileTypes = EGatherTextSourceFileTypes::Ini; } virtual const FString& GetToken() const override { return FPreProcessorDescriptor::IniNamespaceString; } virtual void TryParse(const FString& Text, FSourceFileParseContext& Context) const override; }; static const FString ChangelistName; static FString UnescapeLiteralCharacterEscapeSequences(const FString& InString); static FString RemoveStringFromTextMacro(const FString& TextMacro, const FString& IdentForLogging, bool& Error); static FString StripCommentsFromToken(const FString& InToken, FSourceFileParseContext& Context); static bool ParseSourceText(const FString& Text, const TArray& Parsables, FSourceFileParseContext& ParseCtxt); public: //~ Begin UCommandlet Interface virtual int32 Main(const FString& Params) override; //~ End UCommandlet Interface #undef LOC_DEFINE_REGION };