// Copyright Epic Games, Inc. All Rights Reserved.

#include "Commandlets/GatherTextFromSourceCommandlet.h"
#include "HAL/FileManager.h"
#include "Misc/FileHelper.h"
#include "Misc/Paths.h"
#include "Misc/ExpressionParserTypes.h"
#include "Algo/Transform.h"
#include "Internationalization/InternationalizationMetadata.h"
#include "Internationalization/TextNamespaceUtil.h"

DEFINE_LOG_CATEGORY_STATIC(LogGatherTextFromSourceCommandlet, Log, All);

//////////////////////////////////////////////////////////////////////////
// GatherTextFromSourceCommandlet

#define LOC_DEFINE_REGION

UGatherTextFromSourceCommandlet::UGatherTextFromSourceCommandlet(const FObjectInitializer& ObjectInitializer)
    : Super(ObjectInitializer)
{
}

const FString UGatherTextFromSourceCommandlet::FPreProcessorDescriptor::DefineString(TEXT("#define "));
const FString UGatherTextFromSourceCommandlet::FPreProcessorDescriptor::UndefString(TEXT("#undef "));
const FString UGatherTextFromSourceCommandlet::FPreProcessorDescriptor::IfString(TEXT("#if "));
const FString UGatherTextFromSourceCommandlet::FPreProcessorDescriptor::IfDefString(TEXT("#ifdef "));
const FString UGatherTextFromSourceCommandlet::FPreProcessorDescriptor::ElIfString(TEXT("#elif "));
const FString UGatherTextFromSourceCommandlet::FPreProcessorDescriptor::ElseString(TEXT("#else"));
const FString UGatherTextFromSourceCommandlet::FPreProcessorDescriptor::EndIfString(TEXT("#endif"));
const FString UGatherTextFromSourceCommandlet::FPreProcessorDescriptor::DefinedString(TEXT("defined "));
const FString UGatherTextFromSourceCommandlet::FPreProcessorDescriptor::IniNamespaceString(TEXT("["));
const FString UGatherTextFromSourceCommandlet::FMacroDescriptor::TextMacroString(TEXT("TEXT"));
const FString UGatherTextFromSourceCommandlet::ChangelistName(TEXT("Update Localization"));

int32 UGatherTextFromSourceCommandlet::Main(const FString& Params)
{
    // Parse command line - we're interested in the param vals
    TArray<FString> Tokens;
    TArray<FString> Switches;
    TMap<FString, FString> ParamVals;
    UCommandlet::ParseCommandLine(*Params, Tokens, Switches, ParamVals);

    // Set config file
    const FString* ParamVal = ParamVals.Find(FString(TEXT("Config")));
    FString GatherTextConfigPath;

    if (ParamVal)
    {
        GatherTextConfigPath = *ParamVal;
    }
    else
    {
        UE_LOG(LogGatherTextFromSourceCommandlet, Error, TEXT("No config specified."));
        return -1;
    }

    // Set config section
    ParamVal = ParamVals.Find(FString(TEXT("Section")));
    FString SectionName;

    if (ParamVal)
    {
        SectionName = *ParamVal;
    }
    else
    {
        UE_LOG(LogGatherTextFromSourceCommandlet, Error, TEXT("No config section specified."));
        return -1;
    }

    // SearchDirectoryPaths
    TArray<FString> SearchDirectoryPaths;
    GetPathArrayFromConfig(*SectionName, TEXT("SearchDirectoryPaths"), SearchDirectoryPaths, GatherTextConfigPath);

    // IncludePaths (DEPRECATED)
    {
        TArray<FString> IncludePaths;
        GetPathArrayFromConfig(*SectionName, TEXT("IncludePaths"), IncludePaths, GatherTextConfigPath);
        if (IncludePaths.Num())
        {
            SearchDirectoryPaths.Append(IncludePaths);
            UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("IncludePaths detected in section %s. IncludePaths is deprecated, please use SearchDirectoryPaths."), *SectionName);
        }
    }

    if (SearchDirectoryPaths.Num() == 0)
    {
        UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("No search directory paths in section %s."), *SectionName);
        return 0;
    }

    // ExcludePathFilters
    TArray<FString> ExcludePathFilters;
    GetPathArrayFromConfig(*SectionName, TEXT("ExcludePathFilters"), ExcludePathFilters, GatherTextConfigPath);

    // ExcludePaths (DEPRECATED)
    {
        TArray<FString> ExcludePaths;
        GetPathArrayFromConfig(*SectionName, TEXT("ExcludePaths"), ExcludePaths, GatherTextConfigPath);
        if (ExcludePaths.Num())
        {
            ExcludePathFilters.Append(ExcludePaths);
            UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("ExcludePaths detected in section %s. ExcludePaths is deprecated, please use ExcludePathFilters."), *SectionName);
        }
    }

    // FileNameFilters
    TArray<FString> FileNameFilters;
    GetStringArrayFromConfig(*SectionName, TEXT("FileNameFilters"), FileNameFilters, GatherTextConfigPath);

    // SourceFileSearchFilters (DEPRECATED)
    {
        TArray<FString> SourceFileSearchFilters;
        GetStringArrayFromConfig(*SectionName, TEXT("SourceFileSearchFilters"), SourceFileSearchFilters, GatherTextConfigPath);
        if (SourceFileSearchFilters.Num())
        {
            FileNameFilters.Append(SourceFileSearchFilters);
            UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("SourceFileSearchFilters detected in section %s. SourceFileSearchFilters is deprecated, please use FileNameFilters."), *SectionName);
        }
    }

    if (FileNameFilters.Num() == 0)
    {
        UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("No source filters in section %s"), *SectionName);
        return 0;
    }

    // Ensure all filters are unique.
    TArray<FString> UniqueSourceFileSearchFilters;
    for (const FString& SourceFileSearchFilter: FileNameFilters)
    {
        UniqueSourceFileSearchFilters.AddUnique(SourceFileSearchFilter);
    }

    // Build the final set of include/exclude paths to scan.
    TArray<FString> IncludePathFilters;
    Algo::Transform(SearchDirectoryPaths, IncludePathFilters, [](const FString& SearchDirectoryPath)
                    {
                        return SearchDirectoryPath / TEXT("*");
                    });

    FGatherTextDelegates::GetAdditionalGatherPaths.Broadcast(GatherManifestHelper->GetTargetName(), IncludePathFilters, ExcludePathFilters);

    // Search in the root folder for each of the wildcard filters specified and build a list of files
    TArray<FString> FilesToProcess;
    {
        TArray<FString> RootSourceFiles;
        for (const FString& IncludePathFilter: IncludePathFilters)
        {
            FString SearchDirectoryPath = IncludePathFilter;
            if (SearchDirectoryPath.EndsWith(TEXT("*"), ESearchCase::CaseSensitive))
            {
                // Trim the wildcard from this search path
                SearchDirectoryPath = FPaths::GetPath(MoveTemp(SearchDirectoryPath));
            }

            for (const FString& UniqueSourceFileSearchFilter: UniqueSourceFileSearchFilters)
            {
                IFileManager::Get().FindFilesRecursive(RootSourceFiles, *SearchDirectoryPath, *UniqueSourceFileSearchFilter, true, false, false);

                for (FString& RootSourceFile: RootSourceFiles)
                {
                    if (FPaths::IsRelative(RootSourceFile))
                    {
                        RootSourceFile = FPaths::ConvertRelativePathToFull(MoveTemp(RootSourceFile));
                    }
                }

                FilesToProcess.Append(MoveTemp(RootSourceFiles));
                RootSourceFiles.Reset();
            }
        }
    }

    const FFuzzyPathMatcher FuzzyPathMatcher = FFuzzyPathMatcher(IncludePathFilters, ExcludePathFilters);
    FilesToProcess.RemoveAll([&FuzzyPathMatcher](const FString& FoundFile)
                             {
                                 // Filter out assets whose package file paths do not pass the "fuzzy path" filters.
                                 if (FuzzyPathMatcher.TestPath(FoundFile) != FFuzzyPathMatcher::Included)
                                 {
                                     return true;
                                 }

                                 return false;
                             });

    // Return if no source files were found
    if (FilesToProcess.Num() == 0)
    {
        FString SpecifiedDirectoriesString;
        for (const FString& IncludePath: IncludePathFilters)
        {
            SpecifiedDirectoriesString.Append(FString(SpecifiedDirectoriesString.IsEmpty() ? TEXT("") : TEXT("\n")) + FString::Printf(TEXT("+ %s"), *IncludePath));
        }
        for (const FString& ExcludePath: ExcludePathFilters)
        {
            SpecifiedDirectoriesString.Append(FString(SpecifiedDirectoriesString.IsEmpty() ? TEXT("") : TEXT("\n")) + FString::Printf(TEXT("- %s"), *ExcludePath));
        }

        FString SourceFileSearchFiltersString;
        for (const FString& Filter: UniqueSourceFileSearchFilters)
        {
            SourceFileSearchFiltersString += FString(SourceFileSearchFiltersString.IsEmpty() ? TEXT("") : TEXT(", ")) + Filter;
        }

        UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("The GatherTextFromSource commandlet couldn't find any source files matching (%s) in the specified directories:\n%s"), *SourceFileSearchFiltersString, *SpecifiedDirectoriesString);
        return 0;
    }

    // Add any manifest dependencies if they were provided
    TArray<FString> ManifestDependenciesList;
    GetPathArrayFromConfig(*SectionName, TEXT("ManifestDependencies"), ManifestDependenciesList, GatherTextConfigPath);

    for (const FString& ManifestDependency: ManifestDependenciesList)
    {
        FText OutError;
        if (!GatherManifestHelper->AddDependency(ManifestDependency, &OutError))
        {
            UE_LOG(LogGatherTextFromSourceCommandlet, Error, TEXT("The GatherTextFromSource commandlet couldn't load the specified manifest dependency: '%'. %s"), *ManifestDependency, *OutError.ToString());
            return -1;
        }
    }

    // Get the loc macros and their syntax
    TArray<FParsableDescriptor*> Parsables;

    Parsables.Add(new FDefineDescriptor());

    Parsables.Add(new FUndefDescriptor());

    Parsables.Add(new FIfDescriptor());

    Parsables.Add(new FIfDefDescriptor());

    Parsables.Add(new FElIfDescriptor());

    Parsables.Add(new FElseDescriptor());

    Parsables.Add(new FEndIfDescriptor());

    Parsables.Add(new FUICommandMacroDescriptor());

    Parsables.Add(new FUICommandExtMacroDescriptor());

    // New Localization System with Namespace as literal argument.
    Parsables.Add(new FStringMacroDescriptor(FString(TEXT("NSLOCTEXT")),
                                             FStringMacroDescriptor::FMacroArg(FStringMacroDescriptor::MAS_Namespace, true),
                                             FStringMacroDescriptor::FMacroArg(FStringMacroDescriptor::MAS_Identifier, true),
                                             FStringMacroDescriptor::FMacroArg(FStringMacroDescriptor::MAS_SourceText, true)));

    // New Localization System with Namespace as preprocessor define.
    Parsables.Add(new FStringMacroDescriptor(FString(TEXT("LOCTEXT")),
                                             FStringMacroDescriptor::FMacroArg(FStringMacroDescriptor::MAS_Identifier, true),
                                             FStringMacroDescriptor::FMacroArg(FStringMacroDescriptor::MAS_SourceText, true)));

    Parsables.Add(new FStringTableMacroDescriptor());

    Parsables.Add(new FStringTableFromFileMacroDescriptor(TEXT("LOCTABLE_FROMFILE_ENGINE"), FPaths::EngineContentDir()));

    Parsables.Add(new FStringTableFromFileMacroDescriptor(TEXT("LOCTABLE_FROMFILE_GAME"), FPaths::ProjectContentDir()));

    Parsables.Add(new FStringTableEntryMacroDescriptor());

    Parsables.Add(new FStringTableEntryMetaDataMacroDescriptor());

    Parsables.Add(new FIniNamespaceDescriptor());

    // Init a parse context to track the state of the file parsing
    FSourceFileParseContext ParseCtxt(this);

    // Get whether we should gather editor-only data. Typically only useful for the localization of UE4 itself.
    if (!GetBoolFromConfig(*SectionName, TEXT("ShouldGatherFromEditorOnlyData"), ParseCtxt.ShouldGatherFromEditorOnlyData, GatherTextConfigPath))
    {
        ParseCtxt.ShouldGatherFromEditorOnlyData = false;
    }

    // Parse all source files for macros and add entries to SourceParsedEntries
    for (FString& SourceFile: FilesToProcess)
    {
        FString ProjectBasePath;
        if (!FPaths::ProjectDir().IsEmpty())
        {
            ProjectBasePath = FPaths::ProjectDir();
        }
        else
        {
            ProjectBasePath = FPaths::EngineDir();
        }

        ParseCtxt.Filename  = SourceFile;
        ParseCtxt.FileTypes = ParseCtxt.Filename.EndsWith(TEXT(".ini")) ? EGatherTextSourceFileTypes::Ini : EGatherTextSourceFileTypes::Cpp;
        FPaths::MakePathRelativeTo(ParseCtxt.Filename, *ProjectBasePath);
        ParseCtxt.LineNumber       = 0;
        ParseCtxt.FilePlatformName = GetSplitPlatformNameFromPath(ParseCtxt.Filename);
        ParseCtxt.LineText.Reset();
        ParseCtxt.Namespace.Reset();
        ParseCtxt.RawStringLiteralClosingDelim.Reset();
        ParseCtxt.ExcludedRegion                  = false;
        ParseCtxt.WithinBlockComment              = false;
        ParseCtxt.WithinLineComment               = false;
        ParseCtxt.WithinStringLiteral             = false;
        ParseCtxt.WithinNamespaceDefineLineNumber = INDEX_NONE;
        ParseCtxt.WithinStartingLine              = nullptr;
        ParseCtxt.FlushMacroStack();

        FString SourceFileText;
        if (!FFileHelper::LoadFileToString(SourceFileText, *SourceFile))
        {
            UE_LOG(LogGatherTextFromSourceCommandlet, Error, TEXT("GatherTextSource failed to open file %s"), *ParseCtxt.Filename);
        }
        else
        {
            if (!ParseSourceText(SourceFileText, Parsables, ParseCtxt))
            {
                UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("GatherTextSource error(s) parsing source file %s"), *ParseCtxt.Filename);
            }
            else
            {
                if (ParseCtxt.WithinNamespaceDefineLineNumber != INDEX_NONE)
                {
                    UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("Missing '#undef LOCTEXT_NAMESPACE' for '#define LOCTEXT_NAMESPACE' at %s:%d"), *ParseCtxt.Filename, ParseCtxt.WithinNamespaceDefineLineNumber);
                }
            }
        }
    }

    // Process any parsed string tables
    for (const auto& ParsedStringTablePair: ParseCtxt.ParsedStringTables)
    {
        if (ParsedStringTablePair.Value.SourceLocation.Line == INDEX_NONE)
        {
            UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("String table with ID '%s' had %s entries parsed for it, but the table was never registered. Skipping for gather."), *ParsedStringTablePair.Key.ToString(), ParsedStringTablePair.Value.TableEntries.Num());
        }
        else
        {
            for (const auto& ParsedStringTableEntryPair: ParsedStringTablePair.Value.TableEntries)
            {
                if (!ParsedStringTableEntryPair.Value.bIsEditorOnly || ParseCtxt.ShouldGatherFromEditorOnlyData)
                {
                    FManifestContext SourceContext;
                    SourceContext.Key                                           = ParsedStringTableEntryPair.Key;
                    SourceContext.SourceLocation                                = ParsedStringTableEntryPair.Value.SourceLocation.ToString();
                    SourceContext.PlatformName                                  = ParsedStringTableEntryPair.Value.PlatformName;

                    const FParsedStringTableEntryMetaDataMap* ParsedMetaDataMap = ParsedStringTablePair.Value.MetaDataEntries.Find(ParsedStringTableEntryPair.Key);
                    if (ParsedMetaDataMap && ParsedMetaDataMap->Num() > 0)
                    {
                        SourceContext.InfoMetadataObj = MakeShareable(new FLocMetadataObject());
                        for (const auto& ParsedMetaDataPair: *ParsedMetaDataMap)
                        {
                            if (!ParsedMetaDataPair.Value.bIsEditorOnly || ParseCtxt.ShouldGatherFromEditorOnlyData)
                            {
                                SourceContext.InfoMetadataObj->SetStringField(ParsedMetaDataPair.Key.ToString(), ParsedMetaDataPair.Value.MetaData);
                            }
                        }
                    }

                    GatherManifestHelper->AddSourceText(ParsedStringTablePair.Value.TableNamespace, FLocItem(ParsedStringTableEntryPair.Value.SourceString), SourceContext);
                }
            }
        }
    }

    // Clear parsables list safely
    for (int32 i = 0; i < Parsables.Num(); i++)
    {
        delete Parsables[i];
    }

    return 0;
}

FString UGatherTextFromSourceCommandlet::UnescapeLiteralCharacterEscapeSequences(const FString& InString)
{
    // We need to un-escape any octal, hex, or universal character sequences that exist in this string to mimic what happens when the string is processed by the compiler
    enum class EParseState : uint8
    {
        Idle,     // Not currently parsing a sequence
        InOct,    // Within an octal sequence (\012)
        InHex,    // Within an hexadecimal sequence (\xBEEF)
        InUTF16,  // Within a UTF-16 sequence (\u1234)
        InUTF32,  // Within a UTF-32 sequence (\U12345678)
    };

    FString RetString;
    RetString.Reserve(InString.Len());

    EParseState ParseState = EParseState::Idle;
    FString EscapedLiteralCharacter;
    for (const TCHAR* CharPtr = *InString; *CharPtr; ++CharPtr)
    {
        const TCHAR CurChar = *CharPtr;

        switch (ParseState)
        {
            case EParseState::Idle: {
                const TCHAR NextChar = *(CharPtr + 1);
                if (CurChar == TEXT('\\') && NextChar)
                {
                    if (FChar::IsOctDigit(NextChar))
                    {
                        ParseState = EParseState::InOct;
                    }
                    else if (NextChar == TEXT('x'))
                    {
                        // Skip the format marker
                        ++CharPtr;
                        ParseState = EParseState::InHex;
                    }
                    else if (NextChar == TEXT('u'))
                    {
                        // Skip the format marker
                        ++CharPtr;
                        ParseState = EParseState::InUTF16;
                    }
                    else if (NextChar == TEXT('U'))
                    {
                        // Skip the format marker
                        ++CharPtr;
                        ParseState = EParseState::InUTF32;
                    }
                }

                if (ParseState == EParseState::Idle)
                {
                    RetString.AppendChar(CurChar);
                }
                else
                {
                    EscapedLiteralCharacter.Reset();
                }
            }
            break;

            case EParseState::InOct: {
                if (FChar::IsOctDigit(CurChar))
                {
                    EscapedLiteralCharacter.AppendChar(CurChar);

                    // Octal sequences can only be up-to 3 digits long
                    check(EscapedLiteralCharacter.Len() <= 3);
                    if (EscapedLiteralCharacter.Len() == 3)
                    {
                        RetString.AppendChar((TCHAR)FCString::Strtoi(*EscapedLiteralCharacter, nullptr, 8));
                        ParseState = EParseState::Idle;
                        // Deliberately not appending the current character here, as it was already pushed into the escaped literal character string
                    }
                }
                else
                {
                    RetString.AppendChar((TCHAR)FCString::Strtoi(*EscapedLiteralCharacter, nullptr, 8));
                    ParseState = EParseState::Idle;
                    RetString.AppendChar(CurChar);
                }
            }
            break;

            case EParseState::InHex: {
                if (FChar::IsHexDigit(CurChar))
                {
                    EscapedLiteralCharacter.AppendChar(CurChar);
                }
                else
                {
                    RetString.AppendChar((TCHAR)FCString::Strtoi(*EscapedLiteralCharacter, nullptr, 16));
                    ParseState = EParseState::Idle;
                    RetString.AppendChar(CurChar);
                }
            }
            break;

            case EParseState::InUTF16: {
                if (FChar::IsHexDigit(CurChar))
                {
                    EscapedLiteralCharacter.AppendChar(CurChar);

                    // UTF-16 sequences can only be up-to 4 digits long
                    check(EscapedLiteralCharacter.Len() <= 4);
                    if (EscapedLiteralCharacter.Len() == 4)
                    {
                        const uint32 UnicodeCodepoint = (uint32)FCString::Strtoi(*EscapedLiteralCharacter, nullptr, 16);

                        FString UnicodeString;
                        if (FUnicodeChar::CodepointToString(UnicodeCodepoint, UnicodeString))
                        {
                            RetString.Append(MoveTemp(UnicodeString));
                        }

                        ParseState = EParseState::Idle;
                        // Deliberately not appending the current character here, as it was already pushed into the escaped literal character string
                    }
                }
                else
                {
                    const uint32 UnicodeCodepoint = (uint32)FCString::Strtoi(*EscapedLiteralCharacter, nullptr, 16);

                    FString UnicodeString;
                    if (FUnicodeChar::CodepointToString(UnicodeCodepoint, UnicodeString))
                    {
                        RetString.Append(MoveTemp(UnicodeString));
                    }

                    ParseState = EParseState::Idle;
                    RetString.AppendChar(CurChar);
                }
            }
            break;

            case EParseState::InUTF32: {
                if (FChar::IsHexDigit(CurChar))
                {
                    EscapedLiteralCharacter.AppendChar(CurChar);

                    // UTF-32 sequences can only be up-to 8 digits long
                    check(EscapedLiteralCharacter.Len() <= 8);
                    if (EscapedLiteralCharacter.Len() == 8)
                    {
                        const uint32 UnicodeCodepoint = (uint32)FCString::Strtoui64(*EscapedLiteralCharacter, nullptr, 16);

                        FString UnicodeString;
                        if (FUnicodeChar::CodepointToString(UnicodeCodepoint, UnicodeString))
                        {
                            RetString.Append(MoveTemp(UnicodeString));
                        }

                        ParseState = EParseState::Idle;
                        // Deliberately not appending the current character here, as it was already pushed into the escaped literal character string
                    }
                }
                else
                {
                    const uint32 UnicodeCodepoint = (uint32)FCString::Strtoui64(*EscapedLiteralCharacter, nullptr, 16);

                    FString UnicodeString;
                    if (FUnicodeChar::CodepointToString(UnicodeCodepoint, UnicodeString))
                    {
                        RetString.Append(MoveTemp(UnicodeString));
                    }

                    ParseState = EParseState::Idle;
                    RetString.AppendChar(CurChar);
                }
            }
            break;

            default:
                break;
        }
    }

    return RetString.ReplaceEscapedCharWithChar();
}

FString UGatherTextFromSourceCommandlet::RemoveStringFromTextMacro(const FString& TextMacro, const FString& IdentForLogging, bool& Error)
{
    FString Text;
    Error = true;

    // need to strip text literal out of TextMacro ( format should be TEXT("stringvalue") )
    if (!TextMacro.StartsWith(FMacroDescriptor::TextMacroString))
    {
        Error = false;
        Text  = TextMacro.TrimQuotes();
    }
    else
    {
        int32 OpenQuoteIdx = TextMacro.Find(TEXT("\""), ESearchCase::CaseSensitive);
        if (0 > OpenQuoteIdx || TextMacro.Len() - 1 == OpenQuoteIdx)
        {
            UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("Missing quotes in %s"), *FLocTextHelper::SanitizeLogOutput(IdentForLogging));
        }
        else
        {
            int32 CloseQuoteIdx = TextMacro.Find(TEXT("\""), ESearchCase::CaseSensitive, ESearchDir::FromStart, OpenQuoteIdx + 1);
            if (0 > CloseQuoteIdx)
            {
                UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("Missing quotes in %s"), *FLocTextHelper::SanitizeLogOutput(IdentForLogging));
            }
            else
            {
                Text  = TextMacro.Mid(OpenQuoteIdx + 1, CloseQuoteIdx - OpenQuoteIdx - 1);
                Error = false;
            }
        }
    }

    if (!Error)
    {
        Text = UnescapeLiteralCharacterEscapeSequences(Text);
    }

    return Text;
}

FString UGatherTextFromSourceCommandlet::StripCommentsFromToken(const FString& InToken, FSourceFileParseContext& Context)
{
    check(!Context.WithinBlockComment);
    check(!Context.WithinLineComment);
    check(!Context.WithinStringLiteral);

    // Remove both block and inline comments from the given token
    FString StrippedToken;
    StrippedToken.Reserve(InToken.Len());

    TCHAR WithinQuote     = 0;
    bool bIgnoreNextQuote = false;
    for (const TCHAR* Char = *InToken; *Char; ++Char)
    {
        if (WithinQuote != 0)
        {
            StrippedToken += *Char;

            if (!bIgnoreNextQuote)
            {
                if (*Char == TEXT('\\'))
                {
                    bIgnoreNextQuote = true;
                    continue;
                }

                if (*Char == WithinQuote)
                {
                    // Found an unescaped closing quote - we are no longer within quotes
                    WithinQuote = 0;
                }
            }

            bIgnoreNextQuote = false;
        }
        else
        {
            if (*Char == TEXT('/'))
            {
                const TCHAR* NextChar = Char + 1;

                if (*NextChar == TEXT('/'))
                {
                    // Found an inline quote - this strips the remainder of the string so just break out of the loop
                    break;
                }

                if (*NextChar == TEXT('*'))
                {
                    // Found a block comment - skip all characters until we find the closing quote
                    Context.WithinBlockComment = true;
                    ++Char;  // Skip over the opening slash, and the for loop will skip over the *
                    continue;
                }
            }

            if (Context.WithinBlockComment)
            {
                if (*Char == TEXT('*'))
                {
                    const TCHAR* NextChar = Char + 1;

                    if (*NextChar == TEXT('/'))
                    {
                        // Found the end of a block comment
                        Context.WithinBlockComment = false;
                        ++Char;  // Skip over the opening *, and the for loop will skip over the slash
                        continue;
                    }
                }

                // Skip over all characters while within a block comment
                continue;
            }

            StrippedToken += *Char;

            if (*Char == TEXT('"') || *Char == TEXT('\''))
            {
                // We found an opening quote - keep track of it until we find a matching closing quote
                WithinQuote = *Char;
            }
        }
    }

    return StrippedToken.TrimStartAndEnd();
}

bool UGatherTextFromSourceCommandlet::ParseSourceText(const FString& Text, const TArray<FParsableDescriptor*>& Parsables, FSourceFileParseContext& ParseCtxt)
{
    // Cache array of parsables and tokens valid for this filetype
    TArray<FParsableDescriptor*> ParsablesForFile;
    TArray<FString> ParsableTokensForFile;
    for (FParsableDescriptor* Parsable: Parsables)
    {
        if (Parsable->MatchesFileTypes(ParseCtxt.FileTypes))
        {
            ParsablesForFile.Add(Parsable);
            ParsableTokensForFile.Add(Parsable->GetToken());
        }
    }
    check(ParsablesForFile.Num() == ParsableTokensForFile.Num());

    // Anything to parse for this filetype?
    if (ParsablesForFile.Num() == 0)
    {
        return true;
    }

    // Create array of ints, one for each parsable we're looking for.
    TArray<int32> ParsableMatchCountersForFile;
    ParsableMatchCountersForFile.AddZeroed(ParsablesForFile.Num());

    // Use the file extension to work out what comments look like for this file
    // We default to C++-style comments
    const TCHAR* LineComment       = TEXT("//");
    const TCHAR* BlockCommentStart = TEXT("/*");
    const TCHAR* BlockCommentEnd   = TEXT("*/");
    if (EnumHasAnyFlags(ParseCtxt.FileTypes, EGatherTextSourceFileTypes::Ini))
    {
        LineComment       = TEXT(";");
        BlockCommentStart = nullptr;
        BlockCommentEnd   = nullptr;
    }
    const int32 LineCommentLen       = LineComment ? FCString::Strlen(LineComment) : 0;
    const int32 BlockCommentStartLen = BlockCommentStart ? FCString::Strlen(BlockCommentStart) : 0;
    const int32 BlockCommentEndLen   = BlockCommentEnd ? FCString::Strlen(BlockCommentEnd) : 0;
    checkf((BlockCommentStartLen == 0 && BlockCommentEndLen == 0) || (BlockCommentStartLen > 0 && BlockCommentEndLen > 0), TEXT("Block comments require both a start and an end marker!"));

    // Split the file into lines
    TArray<FString> TextLines;
    Text.ParseIntoArrayLines(TextLines, false);

    // Move through the text lines looking for the tokens that denote the items in the Parsables list
    for (int32 LineIdx = 0; LineIdx < TextLines.Num(); LineIdx++)
    {
        TextLines[LineIdx].TrimEndInline();
        const FString& Line = TextLines[LineIdx];
        if (Line.IsEmpty())
            continue;

        // Use these pending vars to defer parsing a token hit until longer tokens can't hit too
        int32 PendingParseIdx   = INDEX_NONE;
        const TCHAR* ParsePoint = NULL;
        for (int32& Element: ParsableMatchCountersForFile)
        {
            Element = 0;
        }
        ParseCtxt.LineNumber            = LineIdx + 1;
        ParseCtxt.LineText              = Line;
        ParseCtxt.WithinLineComment     = false;
        ParseCtxt.EndParsingCurrentLine = false;

        const TCHAR* Cursor             = *Line;
        while (*Cursor && !ParseCtxt.EndParsingCurrentLine)
        {
            // Check if we're starting comments or string literals

            if (!ParseCtxt.WithinLineComment && !ParseCtxt.WithinBlockComment && !ParseCtxt.WithinStringLiteral)
            {
                if (LineCommentLen > 0 && FCString::Strncmp(Cursor, LineComment, LineCommentLen) == 0)
                {
                    ParseCtxt.WithinLineComment     = true;
                    ParseCtxt.WithinStartingLine    = *Line;
                    ParseCtxt.EndParsingCurrentLine = true;
                    Cursor += LineCommentLen;
                    continue;
                }
                else if (BlockCommentStartLen > 0 && FCString::Strncmp(Cursor, BlockCommentStart, BlockCommentStartLen) == 0)
                {
                    ParseCtxt.WithinBlockComment = true;
                    ParseCtxt.WithinStartingLine = *Line;
                    Cursor += BlockCommentStartLen;
                    continue;
                }
            }

            if (!ParseCtxt.WithinLineComment && !ParseCtxt.WithinBlockComment && !ParseCtxt.WithinStringLiteral)
            {
                if (*Cursor == TEXT('\"'))
                {
                    if (Cursor == *Line)
                    {
                        ParseCtxt.WithinStringLiteral = true;
                        ParseCtxt.WithinStartingLine  = *Line;
                        ++Cursor;
                        continue;
                    }
                    else if (Cursor > *Line)
                    {
                        const TCHAR* const ReverseCursor = Cursor - 1;
                        if (EnumHasAnyFlags(ParseCtxt.FileTypes, EGatherTextSourceFileTypes::Cpp) && *ReverseCursor == TEXT('R'))
                        {
                            // Potentially a C++11 raw string literal, so walk forwards and validate that this looks legit
                            // While doing this we can parse out its optional user defined delimiter so we can find when the string closes
                            //   eg) For 'R"Delim(string)Delim"', ')Delim' would be the closing delimiter.
                            //   eg) For 'R"(string)"', ')' would be the closing delimiter.
                            ParseCtxt.RawStringLiteralClosingDelim = TEXT(")");
                            {
                                bool bIsValid              = true;

                                const TCHAR* ForwardCursor = Cursor + 1;
                                for (;;)
                                {
                                    const TCHAR DelimChar = *ForwardCursor++;
                                    if (DelimChar == TEXT('('))
                                    {
                                        break;
                                    }
                                    if (DelimChar == 0 || !FChar::IsAlnum(DelimChar))
                                    {
                                        bIsValid = false;
                                        break;
                                    }
                                    ParseCtxt.RawStringLiteralClosingDelim += DelimChar;
                                }

                                if (bIsValid)
                                {
                                    ParseCtxt.WithinStringLiteral = true;
                                    ParseCtxt.WithinStartingLine  = *Line;
                                    Cursor                        = ForwardCursor;
                                    continue;
                                }
                                else
                                {
                                    ParseCtxt.RawStringLiteralClosingDelim.Reset();
                                    // Fall through to the quoted string parsing below
                                }
                            }
                        }

                        if (*ReverseCursor != TEXT('\\') && *ReverseCursor != TEXT('\''))
                        {
                            ParseCtxt.WithinStringLiteral = true;
                            ParseCtxt.WithinStartingLine  = *Line;
                            ++Cursor;
                            continue;
                        }
                        else
                        {
                            bool IsEscaped = false;
                            {
                                // if the backslash or single quote is itself escaped then the quote is good
                                const TCHAR* EscapeCursor = ReverseCursor;
                                while (EscapeCursor > *Line && *(--EscapeCursor) == TEXT('\\'))
                                {
                                    IsEscaped = !IsEscaped;
                                }
                            }

                            if (IsEscaped)
                            {
                                ParseCtxt.WithinStringLiteral = true;
                                ParseCtxt.WithinStartingLine  = *Line;
                                ++Cursor;
                                continue;
                            }
                            else
                            {
                                //   check for '"'
                                const TCHAR* const ForwardCursor = Cursor + 1;
                                if (*ReverseCursor == TEXT('\'') && *ForwardCursor != TEXT('\''))
                                {
                                    ParseCtxt.WithinStringLiteral = true;
                                    ParseCtxt.WithinStartingLine  = *Line;
                                    ++Cursor;
                                    continue;
                                }
                            }
                        }
                    }
                }
            }
            else if (ParseCtxt.WithinStringLiteral)
            {
                if (*Cursor == TEXT('\"'))
                {
                    if (Cursor == *Line && ParseCtxt.RawStringLiteralClosingDelim.IsEmpty())
                    {
                        ParseCtxt.WithinStringLiteral = false;
                        ++Cursor;
                        continue;
                    }
                    else if (Cursor > *Line)
                    {
                        // Is this ending a C++11 raw string literal?
                        if (!ParseCtxt.RawStringLiteralClosingDelim.IsEmpty())
                        {
                            const TCHAR* EndDelimCursor = Cursor - ParseCtxt.RawStringLiteralClosingDelim.Len();
                            if (EndDelimCursor >= *Line && FCString::Strncmp(EndDelimCursor, *ParseCtxt.RawStringLiteralClosingDelim, ParseCtxt.RawStringLiteralClosingDelim.Len()) == 0)
                            {
                                ParseCtxt.RawStringLiteralClosingDelim.Reset();
                                ParseCtxt.WithinStringLiteral = false;
                            }
                            ++Cursor;
                            continue;
                        }

                        const TCHAR* const ReverseCursor = Cursor - 1;
                        if (*ReverseCursor != TEXT('\\') && *ReverseCursor != TEXT('\''))
                        {
                            ParseCtxt.WithinStringLiteral = false;
                            ++Cursor;
                            continue;
                        }
                        else
                        {
                            bool IsEscaped = false;
                            {
                                // if the backslash or single quote is itself escaped then the quote is good
                                const TCHAR* EscapeCursor = ReverseCursor;
                                while (EscapeCursor > *Line && *(--EscapeCursor) == TEXT('\\'))
                                {
                                    IsEscaped = !IsEscaped;
                                }
                            }

                            if (IsEscaped)
                            {
                                ParseCtxt.WithinStringLiteral = false;
                                ++Cursor;
                                continue;
                            }
                            else
                            {
                                //   check for '"'
                                const TCHAR* const ForwardCursor = Cursor + 1;
                                if (*ReverseCursor == TEXT('\'') && *ForwardCursor != TEXT('\''))
                                {
                                    ParseCtxt.WithinStringLiteral = false;
                                    ++Cursor;
                                    continue;
                                }
                            }
                        }
                    }
                }
            }

            // Check if we're ending comments
            if (ParseCtxt.WithinBlockComment && BlockCommentEndLen > 0 && FCString::Strncmp(Cursor, BlockCommentEnd, BlockCommentEndLen) == 0)
            {
                ParseCtxt.WithinBlockComment = false;
                Cursor += BlockCommentEndLen;
                continue;
            }

            for (int32 ParIdx = 0; ParIdx < ParsablesForFile.Num(); ++ParIdx)
            {
                FParsableDescriptor* Parsable = ParsablesForFile[ParIdx];
                const FString& Token          = ParsableTokensForFile[ParIdx];

                if (Token.Len() == ParsableMatchCountersForFile[ParIdx])
                {
                    // already seen this entire token and are looking for longer matches - skip it
                    continue;
                }

                if (*Cursor == Token[ParsableMatchCountersForFile[ParIdx]])
                {
                    // Char at cursor matches the next char in the parsable's identifying token
                    if (Token.Len() == ++(ParsableMatchCountersForFile[ParIdx]))
                    {
                        // don't immediately parse - this parsable has seen its entire token but a longer one could be about to hit too
                        const TCHAR* TokenStart = Cursor + 1 - Token.Len();
                        if (0 > PendingParseIdx || ParsePoint >= TokenStart)
                        {
                            PendingParseIdx = ParIdx;
                            ParsePoint      = TokenStart;
                        }
                    }
                }
                else
                {
                    // Char at cursor doesn't match the next char in the parsable's identifying token
                    // Reset the counter to start of the token
                    ParsableMatchCountersForFile[ParIdx] = 0;
                }
            }

            // Now check PendingParse and only run it if there are no better candidates
            if (PendingParseIdx != INDEX_NONE)
            {
                FParsableDescriptor* PendingParsable = ParsablesForFile[PendingParseIdx];

                bool MustDefer                       = false;  // pending will be deferred if another parsable has a equal and greater number of matched chars
                if (!PendingParsable->OverridesLongerTokens())
                {
                    for (int32 ParIdx = 0; ParIdx < ParsablesForFile.Num(); ++ParIdx)
                    {
                        if (PendingParseIdx != ParIdx && ParsableMatchCountersForFile[ParIdx] >= ParsableTokensForFile[PendingParseIdx].Len())
                        {
                            // a longer token is matching so defer
                            MustDefer = true;
                        }
                    }
                }

                if (!MustDefer)
                {
                    // Do the parse now
                    // TODO: Would be nice if TryParse returned what it consumed, and operated on const TCHAR*
                    PendingParsable->TryParse(FString(ParsePoint), ParseCtxt);
                    for (int32& Element: ParsableMatchCountersForFile)
                    {
                        Element = 0;
                    }
                    PendingParseIdx = INDEX_NONE;
                    ParsePoint      = NULL;
                }
            }

            // Advance cursor
            ++Cursor;
        }

        // Handle a string literal that went beyond a single line
        if (ParseCtxt.WithinStringLiteral)
        {
            if (EnumHasAnyFlags(ParseCtxt.FileTypes, EGatherTextSourceFileTypes::Ini))
            {
                // INI files don't support multi-line literals; always terminate them after ending a line
                ParseCtxt.WithinStringLiteral = false;
            }
            else if (Cursor > *Line && ParseCtxt.RawStringLiteralClosingDelim.IsEmpty())
            {
                // C++ only allows multi-line literals if they're escaped with a trailing slash or within a C++11 raw string literal
                ParseCtxt.WithinStringLiteral = *(Cursor - 1) == TEXT('\\');
            }

            UE_CLOG(!ParseCtxt.WithinStringLiteral, LogGatherTextFromSourceCommandlet, Warning, TEXT("A string literal was not correctly terminated. File %s at line %d, starting line: %s"), *ParseCtxt.Filename, ParseCtxt.LineNumber, ParseCtxt.WithinStartingLine);
        }
    }

    // Handle a string C++11 raw string literal that was never closed as this is likely a false positive that needs to be fixed in the parser
    if (ParseCtxt.WithinStringLiteral && !ParseCtxt.RawStringLiteralClosingDelim.IsEmpty())
    {
        UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("A C++11 raw string literal was not correctly terminated. File %s, starting line: %s"), *ParseCtxt.Filename, ParseCtxt.WithinStartingLine);
    }

    return true;
}

bool UGatherTextFromSourceCommandlet::FSourceFileParseContext::AddManifestText(const FString& Token, const FString& InNamespace, const FString& SourceText, const FManifestContext& Context)
{
    const bool bIsEditorOnly = EvaluateEditorOnlyDefineState() == EEditorOnlyDefineState::Defined;

    if (!bIsEditorOnly || ShouldGatherFromEditorOnlyData)
    {
        const FString EntryDescription = FString::Printf(TEXT("%s macro"), *Token);
        return OwnerCommandlet->GatherManifestHelper->AddSourceText(InNamespace, FLocItem(SourceText), Context, &EntryDescription);
    }

    return false;
}

void UGatherTextFromSourceCommandlet::FSourceFileParseContext::PushMacroBlock(const FString& InBlockCtx)
{
    MacroBlockStack.Push(InBlockCtx);
    CachedEditorOnlyDefineState.Reset();
}

void UGatherTextFromSourceCommandlet::FSourceFileParseContext::PopMacroBlock()
{
    if (MacroBlockStack.Num() > 0)
    {
        MacroBlockStack.Pop(/*bAllowShrinking*/ false);
        CachedEditorOnlyDefineState.Reset();
    }
}

void UGatherTextFromSourceCommandlet::FSourceFileParseContext::FlushMacroStack()
{
    MacroBlockStack.Reset();
    CachedEditorOnlyDefineState.Reset();
}

UGatherTextFromSourceCommandlet::EEditorOnlyDefineState UGatherTextFromSourceCommandlet::FSourceFileParseContext::EvaluateEditorOnlyDefineState() const
{
    if (CachedEditorOnlyDefineState.IsSet())
    {
        return CachedEditorOnlyDefineState.GetValue();
    }

    static const FString WithEditorString         = TEXT("WITH_EDITOR");
    static const FString WithEditorOnlyDataString = TEXT("WITH_EDITORONLY_DATA");

    CachedEditorOnlyDefineState                   = EEditorOnlyDefineState::Undefined;
    for (const FString& BlockCtx: MacroBlockStack)
    {
        if (BlockCtx.Equals(WithEditorString, ESearchCase::CaseSensitive) || BlockCtx.Equals(WithEditorOnlyDataString, ESearchCase::CaseSensitive))
        {
            CachedEditorOnlyDefineState = EEditorOnlyDefineState::Defined;
            break;
        }
    }

    return CachedEditorOnlyDefineState.GetValue();
}

void UGatherTextFromSourceCommandlet::FSourceFileParseContext::SetDefine(const FString& InDefineCtx)
{
    static const FString LocDefRegionString = TEXT("LOC_DEFINE_REGION");
    static const FString LocNamespaceString = TEXT("LOCTEXT_NAMESPACE");

    if (InDefineCtx.Equals(LocDefRegionString, ESearchCase::CaseSensitive))
    {
        // #define LOC_DEFINE_REGION
        if (ExcludedRegion)
        {
            UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("Found a '#define LOC_DEFINE_REGION' within another '#define LOC_DEFINE_REGION' while parsing %s:%d"), *Filename, LineNumber);
        }
        else
        {
            ExcludedRegion = true;
        }
        return;
    }
    else if (!ExcludedRegion)
    {
        if (InDefineCtx.StartsWith(LocNamespaceString, ESearchCase::CaseSensitive) && InDefineCtx.IsValidIndex(LocNamespaceString.Len()) && (FText::IsWhitespace(InDefineCtx[LocNamespaceString.Len()]) || InDefineCtx[LocNamespaceString.Len()] == TEXT('"')))
        {
            // #define LOCTEXT_NAMESPACE <namespace>
            if (WithinNamespaceDefineLineNumber != INDEX_NONE)
            {
                UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("Found a '#define LOCTEXT_NAMESPACE' within another '#define LOCTEXT_NAMESPACE' while parsing %s:%d"), *Filename, LineNumber);
            }
            else
            {
                FString RemainingText = InDefineCtx.RightChop(LocNamespaceString.Len()).TrimStart();

                bool RemoveStringError;
                const FString DefineDesc = FString::Printf(TEXT("%s define at %s:%d"), *RemainingText, *Filename, LineNumber);
                FString NewNamespace     = RemoveStringFromTextMacro(RemainingText, DefineDesc, RemoveStringError);

                if (!RemoveStringError)
                {
                    Namespace                       = MoveTemp(NewNamespace);
                    WithinNamespaceDefineLineNumber = LineNumber;
                }
            }
            return;
        }
    }
}

void UGatherTextFromSourceCommandlet::FSourceFileParseContext::RemoveDefine(const FString& InDefineCtx)
{
    static const FString LocDefRegionString = TEXT("LOC_DEFINE_REGION");
    static const FString LocNamespaceString = TEXT("LOCTEXT_NAMESPACE");

    if (InDefineCtx.Equals(LocDefRegionString, ESearchCase::CaseSensitive))
    {
        // #undef LOC_DEFINE_REGION
        if (!ExcludedRegion)
        {
            UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("Found an '#undef LOC_DEFINE_REGION' without a corresponding '#define LOC_DEFINE_REGION' while parsing %s:%d"), *Filename, LineNumber);
        }
        else
        {
            ExcludedRegion = false;
        }
        return;
    }
    else if (!ExcludedRegion)
    {
        if (InDefineCtx.Equals(LocNamespaceString, ESearchCase::CaseSensitive))
        {
            // #undef LOCTEXT_NAMESPACE
            if (WithinNamespaceDefineLineNumber == INDEX_NONE)
            {
                UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("Found an '#undef LOCTEXT_NAMESPACE' without a corresponding '#define LOCTEXT_NAMESPACE' while parsing %s:%d"), *Filename, LineNumber);
            }
            else
            {
                Namespace.Empty();
                WithinNamespaceDefineLineNumber = INDEX_NONE;
            }
            return;
        }
    }
}

bool UGatherTextFromSourceCommandlet::FSourceFileParseContext::AddStringTableImpl(const FName InTableId, const FString& InTableNamespace)
{
    // String table entries may be parsed before the string table itself (due to code ordering), so only warn about duplication here if we've already got a source location for the string table (as adding entries doesn't set that)
    FParsedStringTable& ParsedStringTable = ParsedStringTables.FindOrAdd(InTableId);
    if (ParsedStringTable.SourceLocation.Line != INDEX_NONE)
    {
        UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("String table with ID \"%s\" at \"%s\" was already parsed at \"%s\". Ignoring additional definition."), *InTableId.ToString(), *FSourceLocation(Filename, LineNumber).ToString(), *ParsedStringTable.SourceLocation.ToString());
        return false;
    }

    ParsedStringTable.TableNamespace = InTableNamespace;
    ParsedStringTable.SourceLocation = FSourceLocation(Filename, LineNumber);
    return true;
}

bool UGatherTextFromSourceCommandlet::FSourceFileParseContext::AddStringTableEntryImpl(const FName InTableId, const FString& InKey, const FString& InSourceString, const FSourceLocation& InSourceLocation, const FName InPlatformName)
{
    const bool bIsEditorOnly = EvaluateEditorOnlyDefineState() == EEditorOnlyDefineState::Defined;

    // String table entries may be parsed before the string table itself (due to code ordering), so we may need to add our string table below
    FParsedStringTable& ParsedStringTable  = ParsedStringTables.FindOrAdd(InTableId);

    FParsedStringTableEntry* ExistingEntry = ParsedStringTable.TableEntries.Find(InKey);
    if (ExistingEntry)
    {
        if (ExistingEntry->SourceString.Equals(InSourceString, ESearchCase::CaseSensitive))
        {
            ExistingEntry->bIsEditorOnly &= bIsEditorOnly;
            return true;
        }
        else
        {
            UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("String table entry with ID \"%s\" and key \"%s\" at \"%s\" was already parsed at \"%s\". Ignoring additional definition."), *InTableId.ToString(), *InKey, *FSourceLocation(Filename, LineNumber).ToString(), *ExistingEntry->SourceLocation.ToString());
            return false;
        }
    }
    else
    {
        FParsedStringTableEntry& ParsedStringTableEntry = ParsedStringTable.TableEntries.Add(InKey);
        ParsedStringTableEntry.SourceString             = InSourceString;
        ParsedStringTableEntry.SourceLocation           = InSourceLocation;
        ParsedStringTableEntry.PlatformName             = InPlatformName;
        ParsedStringTableEntry.bIsEditorOnly            = bIsEditorOnly;
        return true;
    }
}

bool UGatherTextFromSourceCommandlet::FSourceFileParseContext::AddStringTableEntryMetaDataImpl(const FName InTableId, const FString& InKey, const FName InMetaDataId, const FString& InMetaData, const FSourceLocation& InSourceLocation)
{
    const bool bIsEditorOnly = EvaluateEditorOnlyDefineState() == EEditorOnlyDefineState::Defined;

    // String table meta-data may be parsed before the string table itself (due to code ordering), so we may need to add our string table below
    FParsedStringTable& ParsedStringTable             = ParsedStringTables.FindOrAdd(InTableId);
    FParsedStringTableEntryMetaDataMap& MetaDataMap   = ParsedStringTable.MetaDataEntries.FindOrAdd(InKey);

    FParsedStringTableEntryMetaData* ExistingMetaData = MetaDataMap.Find(InMetaDataId);
    if (ExistingMetaData)
    {
        if (ExistingMetaData->MetaData.Equals(InMetaData, ESearchCase::CaseSensitive))
        {
            ExistingMetaData->bIsEditorOnly &= bIsEditorOnly;
            return true;
        }
        else
        {
            UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("String table entry meta-data with ID \"%s\" and key \"%s\" at \"%s\" was already parsed at \"%s\". Ignoring additional definition."), *InTableId.ToString(), *InKey, *FSourceLocation(Filename, LineNumber).ToString(), *ExistingMetaData->SourceLocation.ToString());
            return false;
        }
    }
    else
    {
        FParsedStringTableEntryMetaData& ParsedMetaData = MetaDataMap.Add(InMetaDataId);
        ParsedMetaData.MetaData                         = InMetaData;
        ParsedMetaData.SourceLocation                   = InSourceLocation;
        ParsedMetaData.bIsEditorOnly                    = bIsEditorOnly;
        return true;
    }
}

void UGatherTextFromSourceCommandlet::FSourceFileParseContext::AddStringTable(const FName InTableId, const FString& InTableNamespace)
{
    AddStringTableImpl(InTableId, InTableNamespace);
}

void UGatherTextFromSourceCommandlet::FSourceFileParseContext::AddStringTableFromFile(const FName InTableId, const FString& InTableNamespace, const FString& InTableFilename, const FString& InRootPath)
{
    if (AddStringTableImpl(InTableId, InTableNamespace))
    {
        const FString FullImportPath   = InRootPath / InTableFilename;

        FStringTableRef TmpStringTable = FStringTable::NewStringTable();
        if (TmpStringTable->ImportStrings(FullImportPath))
        {
            const FSourceLocation SourceLocation = FSourceLocation(InTableFilename, INDEX_NONE);
            const FName TablePlatformName        = OwnerCommandlet->GetSplitPlatformNameFromPath(InTableFilename);

            TmpStringTable->EnumerateSourceStrings([&](const FString& InKey, const FString& InSourceString)
                                                   {
                                                       AddStringTableEntryImpl(InTableId, InKey, InSourceString, SourceLocation, TablePlatformName);

                                                       TmpStringTable->EnumerateMetaData(InKey, [&](const FName InMetaDataId, const FString& InMetaData)
                                                                                         {
                                                                                             AddStringTableEntryMetaDataImpl(InTableId, InKey, InMetaDataId, InMetaData, SourceLocation);
                                                                                             return true;  // continue enumeration
                                                                                         });

                                                       return true;  // continue enumeration
                                                   });
        }
        else
        {
            UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("String table with ID \"%s\" at \"%s\" failed to import strings from \"%s\"."), *InTableId.ToString(), *FSourceLocation(Filename, LineNumber).ToString(), *FullImportPath);
        }
    }
}

void UGatherTextFromSourceCommandlet::FSourceFileParseContext::AddStringTableEntry(const FName InTableId, const FString& InKey, const FString& InSourceString)
{
    AddStringTableEntryImpl(InTableId, InKey, InSourceString, FSourceLocation(Filename, LineNumber), FilePlatformName);
}

void UGatherTextFromSourceCommandlet::FSourceFileParseContext::AddStringTableEntryMetaData(const FName InTableId, const FString& InKey, const FName InMetaDataId, const FString& InMetaData)
{
    AddStringTableEntryMetaDataImpl(InTableId, InKey, InMetaDataId, InMetaData, FSourceLocation(Filename, LineNumber));
}

void UGatherTextFromSourceCommandlet::FDefineDescriptor::TryParse(const FString& Text, FSourceFileParseContext& Context) const
{
    // Attempt to parse something of the format
    // #define <defname>
    //  or
    // #define <defname> <value>

    if (!Context.WithinBlockComment && !Context.WithinLineComment && !Context.WithinStringLiteral)
    {
        FString RemainingText = Text.RightChop(GetToken().Len()).TrimStart();
        RemainingText         = StripCommentsFromToken(RemainingText, Context);

        Context.SetDefine(RemainingText);
        Context.EndParsingCurrentLine = true;
    }
}

void UGatherTextFromSourceCommandlet::FUndefDescriptor::TryParse(const FString& Text, FSourceFileParseContext& Context) const
{
    // Attempt to parse something of the format
    // #undef <defname>

    if (!Context.WithinBlockComment && !Context.WithinLineComment && !Context.WithinStringLiteral)
    {
        FString RemainingText = Text.RightChop(GetToken().Len()).TrimStart();
        RemainingText         = StripCommentsFromToken(RemainingText, Context);

        Context.RemoveDefine(RemainingText);
        Context.EndParsingCurrentLine = true;
    }
}

void UGatherTextFromSourceCommandlet::FIfDescriptor::TryParse(const FString& Text, FSourceFileParseContext& Context) const
{
    // Attempt to parse something of the format
    // #if <defname>

    if (!Context.WithinBlockComment && !Context.WithinLineComment && !Context.WithinStringLiteral)
    {
        FString RemainingText = Text.RightChop(GetToken().Len()).TrimStart();
        RemainingText         = StripCommentsFromToken(RemainingText, Context);

        // Handle "#if defined <defname>"
        if (RemainingText.StartsWith(DefinedString, ESearchCase::CaseSensitive))
        {
            RemainingText.RightChopInline(DefinedString.Len(), false);
            RemainingText.TrimStartInline();
        }

        Context.PushMacroBlock(RemainingText);
        Context.EndParsingCurrentLine = true;
    }
}

void UGatherTextFromSourceCommandlet::FIfDefDescriptor::TryParse(const FString& Text, FSourceFileParseContext& Context) const
{
    // Attempt to parse something of the format
    // #ifdef <defname>

    if (!Context.WithinBlockComment && !Context.WithinLineComment && !Context.WithinStringLiteral)
    {
        FString RemainingText = Text.RightChop(GetToken().Len()).TrimStart();
        RemainingText         = StripCommentsFromToken(RemainingText, Context);

        Context.PushMacroBlock(RemainingText);
        Context.EndParsingCurrentLine = true;
    }
}

void UGatherTextFromSourceCommandlet::FElIfDescriptor::TryParse(const FString& Text, FSourceFileParseContext& Context) const
{
    // Attempt to parse something of the format
    // #elif <defname>

    if (!Context.WithinBlockComment && !Context.WithinLineComment && !Context.WithinStringLiteral)
    {
        FString RemainingText = Text.RightChop(GetToken().Len()).TrimStart();
        RemainingText         = StripCommentsFromToken(RemainingText, Context);

        // Handle "#elif defined <defname>"
        if (RemainingText.StartsWith(DefinedString, ESearchCase::CaseSensitive))
        {
            RemainingText.RightChopInline(DefinedString.Len(), false);
            RemainingText.TrimStartInline();
        }

        Context.PopMacroBlock();  // Pop the current #if or #ifdef state
        Context.PushMacroBlock(RemainingText);
        Context.EndParsingCurrentLine = true;
    }
}

void UGatherTextFromSourceCommandlet::FElseDescriptor::TryParse(const FString& Text, FSourceFileParseContext& Context) const
{
    // Attempt to parse something of the format
    // #else

    if (!Context.WithinBlockComment && !Context.WithinLineComment && !Context.WithinStringLiteral)
    {
        Context.PopMacroBlock();  // Pop the current #if or #ifdef state
        Context.PushMacroBlock(FString());
        Context.EndParsingCurrentLine = true;
    }
}

void UGatherTextFromSourceCommandlet::FEndIfDescriptor::TryParse(const FString& Text, FSourceFileParseContext& Context) const
{
    // Attempt to parse something of the format
    // #endif

    if (!Context.WithinBlockComment && !Context.WithinLineComment && !Context.WithinStringLiteral)
    {
        Context.PopMacroBlock();  // Pop the current #if or #ifdef state
        Context.EndParsingCurrentLine = true;
    }
}

bool UGatherTextFromSourceCommandlet::FMacroDescriptor::ParseArgsFromMacro(const FString& Text, TArray<FString>& Args, FSourceFileParseContext& Context) const
{
    // Attempt to parse something of the format
    // NAME(param0, param1, param2, etc)

    bool Success = false;

    // Step over the token name and any whitespace after it
    FString RemainingText = Text.RightChop(GetToken().Len());
    RemainingText.TrimStartInline();

    const int32 OpenBracketIdx = RemainingText.Find(TEXT("("), ESearchCase::CaseSensitive);
    if (OpenBracketIdx == INDEX_NONE)
    {
        // No opening bracket; warn about this, but don't consider it an error as we're likely parsing something we shouldn't be
        UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("Missing bracket '(' in %s macro at %s:%d. %s"), *GetToken(), *Context.Filename, Context.LineNumber, *FLocTextHelper::SanitizeLogOutput(Context.LineText.TrimStartAndEnd()));
        return false;
    }
    else if (OpenBracketIdx > 0)
    {
        // We stepped over the whitespace when building RemainingText, so if the bracket isn't the first character in the text then it means we only partially matched a longer token and shouldn't parse it
        return false;
    }
    else
    {
        Args.Empty();

        bool bInDblQuotes     = false;
        bool bInSglQuotes     = false;
        int32 BracketStack    = 1;
        bool bEscapeNextChar  = false;

        const TCHAR* ArgStart = *RemainingText + OpenBracketIdx + 1;
        const TCHAR* Cursor   = ArgStart;
        for (; 0 < BracketStack && '\0' != *Cursor; ++Cursor)
        {
            if (bEscapeNextChar)
            {
                bEscapeNextChar = false;
            }
            else if ((bInDblQuotes || bInSglQuotes) && !bEscapeNextChar && '\\' == *Cursor)
            {
                bEscapeNextChar = true;
            }
            else if (bInDblQuotes)
            {
                if ('\"' == *Cursor)
                {
                    bInDblQuotes = false;
                }
            }
            else if (bInSglQuotes)
            {
                if ('\'' == *Cursor)
                {
                    bInSglQuotes = false;
                }
            }
            else if ('\"' == *Cursor)
            {
                bInDblQuotes = true;
            }
            else if ('\'' == *Cursor)
            {
                bInSglQuotes = true;
            }
            else if ('(' == *Cursor)
            {
                ++BracketStack;
            }
            else if (')' == *Cursor)
            {
                --BracketStack;

                if (0 > BracketStack)
                {
                    UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("Unexpected bracket ')' in %s macro while parsing %s:%d. %s"), *GetToken(), *Context.Filename, Context.LineNumber, *FLocTextHelper::SanitizeLogOutput(Context.LineText.TrimStartAndEnd()));
                    return false;
                }
            }
            else if (1 == BracketStack && ',' == *Cursor)
            {
                // create argument from ArgStart to Cursor and set Start next char
                Args.Add(FString(Cursor - ArgStart, ArgStart));
                ArgStart = Cursor + 1;
            }
        }

        if (0 == BracketStack)
        {
            Args.Add(FString(Cursor - ArgStart - 1, ArgStart));
        }
        // else
        //{
        //	Args.Add(FString(ArgStart));
        // }

        Success = 0 < Args.Num() ? true : false;
    }

    return Success;
}

bool UGatherTextFromSourceCommandlet::FMacroDescriptor::PrepareArgument(FString& Argument, bool IsAutoText, const FString& IdentForLogging, bool& OutHasQuotes)
{
    bool Error = false;
    if (!IsAutoText)
    {
        Argument     = RemoveStringFromTextMacro(Argument, IdentForLogging, Error);
        OutHasQuotes = Error ? false : true;
    }
    else
    {
        Argument = Argument.TrimEnd().TrimQuotes(&OutHasQuotes);
        Argument = UnescapeLiteralCharacterEscapeSequences(Argument);
    }
    return Error ? false : true;
}

void UGatherTextFromSourceCommandlet::FUICommandMacroDescriptor::TryParseArgs(const FString& Text, FSourceFileParseContext& Context, const TArray<FString>& Arguments, const int32 ArgIndexOffset) const
{
    FString Identifier = Arguments[ArgIndexOffset];
    Identifier.TrimStartInline();

    // Identifier may optionally be in quotes, as it's sometimes a string literal (in UE_COMMAND_EXT), and sometimes stringified by the macro (in UI_COMMAND)
    // Because this is optional, we don't care if this processing fails
    bool HasQuotes = false;
    PrepareArgument(Identifier, true, FString(), HasQuotes);

    FString SourceLocation = FSourceLocation(Context.Filename, Context.LineNumber).ToString();
    if (Identifier.IsEmpty())
    {
        // The command doesn't have an identifier so we can't gather it
        UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("%s macro at %s has an empty identifier and cannot be gathered."), *GetToken(), *SourceLocation);
        return;
    }

    FString SourceText = Arguments[ArgIndexOffset + 1];
    SourceText.TrimStartInline();

    static const FString UICommandRootNamespace = TEXT("UICommands");
    FString Namespace                           = Context.WithinNamespaceDefineLineNumber != INDEX_NONE && !Context.Namespace.IsEmpty() ? FString::Printf(TEXT("%s.%s"), *UICommandRootNamespace, *Context.Namespace) : UICommandRootNamespace;

    // parse DefaultLangString argument - this arg will be in quotes without TEXT macro
    FString MacroDesc = FString::Printf(TEXT("\"FriendlyName\" argument in %s macro at %s:%d."), *GetToken(), *Context.Filename, Context.LineNumber);
    if (PrepareArgument(SourceText, true, MacroDesc, HasQuotes))
    {
        if (HasQuotes && !Identifier.IsEmpty() && !SourceText.IsEmpty())
        {
            // First create the command entry
            FManifestContext CommandContext;
            CommandContext.Key            = Identifier;
            CommandContext.SourceLocation = SourceLocation;
            CommandContext.PlatformName   = Context.FilePlatformName;

            Context.AddManifestText(GetToken(), Namespace, SourceText, CommandContext);

            // parse DefaultLangTooltipString argument - this arg will be in quotes without TEXT macro
            FString TooltipSourceText = Arguments[ArgIndexOffset + 2];
            TooltipSourceText.TrimStartInline();
            MacroDesc = FString::Printf(TEXT("\"InDescription\" argument in %s macro at %s:%d"), *GetToken(), *Context.Filename, Context.LineNumber);
            if (PrepareArgument(TooltipSourceText, true, MacroDesc, HasQuotes))
            {
                if (HasQuotes && !TooltipSourceText.IsEmpty())
                {
                    // Create the tooltip entry
                    FManifestContext CommandTooltipContext;
                    CommandTooltipContext.Key            = Identifier + TEXT("_ToolTip");
                    CommandTooltipContext.SourceLocation = SourceLocation;
                    CommandTooltipContext.PlatformName   = CommandContext.PlatformName;

                    Context.AddManifestText(GetToken(), Namespace, TooltipSourceText, CommandTooltipContext);
                }
            }
        }
    }
}

void UGatherTextFromSourceCommandlet::FUICommandMacroDescriptor::TryParse(const FString& Text, FSourceFileParseContext& Context) const
{
    // Attempt to parse something of the format
    // UI_COMMAND(LocKey, DefaultLangString, DefaultLangTooltipString, ...)

    if (!Context.ExcludedRegion && !Context.WithinBlockComment && !Context.WithinLineComment && !Context.WithinStringLiteral)
    {
        TArray<FString> Arguments;
        if (ParseArgsFromMacro(StripCommentsFromToken(Text, Context), Arguments, Context))
        {
            // Need at least 3 arguments
            if (Arguments.Num() < 3)
            {
                UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("Expected at least 3 arguments for %s macro, but got %d while parsing %s:%d. %s"), *GetToken(), Arguments.Num(), *Context.Filename, Context.LineNumber, *FLocTextHelper::SanitizeLogOutput(Context.LineText.TrimStartAndEnd()));
            }
            else
            {
                TryParseArgs(Text, Context, Arguments, 0);
            }
        }
    }
}

void UGatherTextFromSourceCommandlet::FUICommandExtMacroDescriptor::TryParse(const FString& Text, FSourceFileParseContext& Context) const
{
    // Attempt to parse something of the format
    // UI_COMMAND_EXT(<IgnoredParam>, <IgnoredParam>, LocKey, DefaultLangString, DefaultLangTooltipString, ...)

    if (!Context.ExcludedRegion && !Context.WithinBlockComment && !Context.WithinLineComment && !Context.WithinStringLiteral)
    {
        TArray<FString> Arguments;
        if (ParseArgsFromMacro(StripCommentsFromToken(Text, Context), Arguments, Context))
        {
            // Need at least 5 arguments
            if (Arguments.Num() < 5)
            {
                UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("Expected at least 5 arguments for %s macro, but got %d while parsing %s:%d. %s"), *GetToken(), Arguments.Num(), *Context.Filename, Context.LineNumber, *FLocTextHelper::SanitizeLogOutput(Context.LineText.TrimStartAndEnd()));
            }
            else
            {
                TryParseArgs(Text, Context, Arguments, 2);
            }
        }
    }
}

void UGatherTextFromSourceCommandlet::FStringMacroDescriptor::TryParse(const FString& Text, FSourceFileParseContext& Context) const
{
    // Attempt to parse something of the format
    // MACRONAME(param0, param1, param2)

    if (!Context.ExcludedRegion && !Context.WithinBlockComment && !Context.WithinLineComment && !Context.WithinStringLiteral)
    {
        TArray<FString> ArgArray;
        if (ParseArgsFromMacro(StripCommentsFromToken(Text, Context), ArgArray, Context))
        {
            int32 NumArgs = ArgArray.Num();

            if (NumArgs != Arguments.Num())
            {
                UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("Expected %d arguments for %s macro, but got %d while parsing %s:%d. %s"), Arguments.Num(), *GetToken(), NumArgs, *Context.Filename, Context.LineNumber, *FLocTextHelper::SanitizeLogOutput(Context.LineText.TrimStartAndEnd()));
            }
            else
            {
                FString Identifier;
                FString SourceLocation = FSourceLocation(Context.Filename, Context.LineNumber).ToString();
                FString SourceText;

                TOptional<FString> Namespace;
                if (Context.WithinNamespaceDefineLineNumber != INDEX_NONE || !Context.Namespace.IsEmpty())
                {
                    Namespace = Context.Namespace;
                }

                bool ArgParseError = false;
                for (int32 ArgIdx = 0; ArgIdx < Arguments.Num(); ArgIdx++)
                {
                    FMacroArg Arg = Arguments[ArgIdx];
                    ArgArray[ArgIdx].TrimStartInline();
                    FString ArgText = ArgArray[ArgIdx];

                    bool HasQuotes;
                    FString MacroDesc = FString::Printf(TEXT("argument %d of %d in %s macro at %s:%d"), ArgIdx + 1, Arguments.Num(), *GetToken(), *Context.Filename, Context.LineNumber);
                    if (!PrepareArgument(ArgText, Arg.IsAutoText, MacroDesc, HasQuotes))
                    {
                        ArgParseError = true;
                        break;
                    }

                    switch (Arg.Semantic)
                    {
                        case MAS_Namespace: {
                            Namespace = ArgText;
                        }
                        break;
                        case MAS_Identifier: {
                            Identifier = ArgText;
                        }
                        break;
                        case MAS_SourceText: {
                            SourceText = ArgText;
                        }
                        break;
                    }
                }

                if (Identifier.IsEmpty())
                {
                    // The command doesn't have an identifier so we can't gather it
                    UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("%s macro at %s has an empty identifier and cannot be gathered."), *GetToken(), *SourceLocation);
                    return;
                }

                if (!ArgParseError && !Identifier.IsEmpty() && !SourceText.IsEmpty())
                {
                    if (!Namespace.IsSet())
                    {
                        UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("%s macro at %s doesn't define a namespace and no external namespace was set. An empty namspace will be used."), *GetToken(), *SourceLocation);
                        Namespace = FString();
                    }

                    FManifestContext MacroContext;
                    MacroContext.Key            = Identifier;
                    MacroContext.SourceLocation = SourceLocation;
                    MacroContext.PlatformName   = Context.FilePlatformName;

                    if (EnumHasAnyFlags(Context.FileTypes, EGatherTextSourceFileTypes::Ini))
                    {
                        // Gather the text without its package ID, as the INI will strip it on load at runtime
                        TextNamespaceUtil::StripPackageNamespaceInline(Namespace.GetValue());
                    }

                    Context.AddManifestText(GetToken(), Namespace.GetValue(), SourceText, MacroContext);
                }
            }
        }
    }
}

void UGatherTextFromSourceCommandlet::FStringTableMacroDescriptor::TryParse(const FString& Text, FSourceFileParseContext& Context) const
{
    // Attempt to parse something of the format
    // LOCTABLE_NEW(Id, Namespace)

    if (!Context.ExcludedRegion && !Context.WithinBlockComment && !Context.WithinLineComment && !Context.WithinStringLiteral)
    {
        TArray<FString> Arguments;
        if (ParseArgsFromMacro(StripCommentsFromToken(Text, Context), Arguments, Context))
        {
            if (Arguments.Num() != 2)
            {
                UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("Expected 2 arguments for %s macro, but got %d while parsing %s:%d. %s"), *GetToken(), Arguments.Num(), *Context.Filename, Context.LineNumber, *FLocTextHelper::SanitizeLogOutput(Context.LineText.TrimStartAndEnd()));
            }
            else
            {
                Arguments[0].TrimStartInline();
                FString TableId = Arguments[0];
                Arguments[1].TrimStartInline();
                FString TableNamespace                = Arguments[1];

                const FString TableIdMacroDesc        = FString::Printf(TEXT("\"Id\" argument in %s macro at %s:%d"), *GetToken(), *Context.Filename, Context.LineNumber);
                const FString TableNamespaceMacroDesc = FString::Printf(TEXT("\"Namespace\" argument in %s macro at %s:%d"), *GetToken(), *Context.Filename, Context.LineNumber);

                bool HasQuotes;
                if (PrepareArgument(TableId, true, TableIdMacroDesc, HasQuotes) && PrepareArgument(TableNamespace, true, TableNamespaceMacroDesc, HasQuotes))
                {
                    const FName TableIdName = *TableId;

                    if (TableIdName.IsNone())
                    {
                        UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("%s macro at %s:%d has an empty identifier and cannot be gathered."), *GetToken(), *Context.Filename, Context.LineNumber);
                    }
                    else
                    {
                        Context.AddStringTable(TableIdName, TableNamespace);
                    }
                }
            }
        }
    }
}

void UGatherTextFromSourceCommandlet::FStringTableFromFileMacroDescriptor::TryParse(const FString& Text, FSourceFileParseContext& Context) const
{
    // Attempt to parse something of the format
    // LOCTABLE_FROMFILE_X(Id, Namespace, FilePath)

    if (!Context.ExcludedRegion && !Context.WithinBlockComment && !Context.WithinLineComment && !Context.WithinStringLiteral)
    {
        TArray<FString> Arguments;
        if (ParseArgsFromMacro(StripCommentsFromToken(Text, Context), Arguments, Context))
        {
            if (Arguments.Num() != 3)
            {
                UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("Expected 3 arguments for %s macro, but got %d while parsing %s:%d. %s"), *GetToken(), Arguments.Num(), *Context.Filename, Context.LineNumber, *FLocTextHelper::SanitizeLogOutput(Context.LineText.TrimStartAndEnd()));
            }
            else
            {
                Arguments[0].TrimStartInline();
                FString TableId = Arguments[0];
                Arguments[1].TrimStartInline();
                FString TableNamespace = Arguments[1];
                Arguments[2].TrimStartInline();
                FString TableFilename                 = Arguments[2];

                const FString TableIdMacroDesc        = FString::Printf(TEXT("\"Id\" argument in %s macro at %s:%d"), *GetToken(), *Context.Filename, Context.LineNumber);
                const FString TableNamespaceMacroDesc = FString::Printf(TEXT("\"Namespace\" argument in %s macro at %s:%d"), *GetToken(), *Context.Filename, Context.LineNumber);
                const FString TableFilenameMacroDesc  = FString::Printf(TEXT("\"FilePath\" argument in %s macro at %s:%d"), *GetToken(), *Context.Filename, Context.LineNumber);

                bool HasQuotes;
                if (PrepareArgument(TableId, true, TableIdMacroDesc, HasQuotes) && PrepareArgument(TableNamespace, true, TableNamespaceMacroDesc, HasQuotes) && PrepareArgument(TableFilename, true, TableFilenameMacroDesc, HasQuotes))
                {
                    const FName TableIdName = *TableId;

                    if (TableIdName.IsNone())
                    {
                        UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("%s macro at %s:%d has an empty identifier and cannot be gathered."), *GetToken(), *Context.Filename, Context.LineNumber);
                    }
                    else
                    {
                        Context.AddStringTableFromFile(TableIdName, TableNamespace, TableFilename, RootPath);
                    }
                }
            }
        }
    }
}

void UGatherTextFromSourceCommandlet::FStringTableEntryMacroDescriptor::TryParse(const FString& Text, FSourceFileParseContext& Context) const
{
    // Attempt to parse something of the format
    // LOCTABLE_SETSTRING(Id, Key, SourceString)

    if (!Context.ExcludedRegion && !Context.WithinBlockComment && !Context.WithinLineComment && !Context.WithinStringLiteral)
    {
        TArray<FString> Arguments;
        if (ParseArgsFromMacro(StripCommentsFromToken(Text, Context), Arguments, Context))
        {
            if (Arguments.Num() != 3)
            {
                UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("Expected 3 arguments for %s macro, but got %d while parsing %s:%d. %s"), *GetToken(), Arguments.Num(), *Context.Filename, Context.LineNumber, *FLocTextHelper::SanitizeLogOutput(Context.LineText.TrimStartAndEnd()));
            }
            else
            {
                Arguments[0].TrimStartInline();
                FString TableId = Arguments[0];
                Arguments[1].TrimStartInline();
                FString Key = Arguments[1];
                Arguments[2].TrimStartInline();
                FString SourceString                = Arguments[2];

                const FString TableIdMacroDesc      = FString::Printf(TEXT("\"Id\" argument in %s macro at %s:%d"), *GetToken(), *Context.Filename, Context.LineNumber);
                const FString KeyMacroDesc          = FString::Printf(TEXT("\"Key\" argument in %s macro at %s:%d"), *GetToken(), *Context.Filename, Context.LineNumber);
                const FString SourceStringMacroDesc = FString::Printf(TEXT("\"SourceString\" argument in %s macro at %s:%d"), *GetToken(), *Context.Filename, Context.LineNumber);

                bool HasQuotes;
                if (PrepareArgument(TableId, true, TableIdMacroDesc, HasQuotes) && PrepareArgument(Key, true, KeyMacroDesc, HasQuotes) && PrepareArgument(SourceString, true, SourceStringMacroDesc, HasQuotes))
                {
                    const FName TableIdName = *TableId;

                    if (TableIdName.IsNone() || Key.IsEmpty())
                    {
                        UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("%s macro at %s:%d has an empty identifier and cannot be gathered."), *GetToken(), *Context.Filename, Context.LineNumber);
                    }
                    else if (!SourceString.IsEmpty())
                    {
                        Context.AddStringTableEntry(TableIdName, Key, SourceString);
                    }
                }
            }
        }
    }
}

void UGatherTextFromSourceCommandlet::FStringTableEntryMetaDataMacroDescriptor::TryParse(const FString& Text, FSourceFileParseContext& Context) const
{
    // Attempt to parse something of the format
    // LOCTABLE_SETMETA(Id, Key, SourceString)

    if (!Context.ExcludedRegion && !Context.WithinBlockComment && !Context.WithinLineComment && !Context.WithinStringLiteral)
    {
        TArray<FString> Arguments;
        if (ParseArgsFromMacro(StripCommentsFromToken(Text, Context), Arguments, Context))
        {
            if (Arguments.Num() != 4)
            {
                UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("Expected 4 arguments for %s macro, but got %d while parsing %s:%d. %s"), *GetToken(), Arguments.Num(), *Context.Filename, Context.LineNumber, *FLocTextHelper::SanitizeLogOutput(Context.LineText.TrimStartAndEnd()));
            }
            else
            {
                Arguments[0].TrimStartInline();
                FString TableId = Arguments[0];
                Arguments[1].TrimStartInline();
                FString Key = Arguments[1];
                Arguments[2].TrimStartInline();
                FString MetaDataId = Arguments[2];
                Arguments[3].TrimStartInline();
                FString MetaData                  = Arguments[3];

                const FString TableIdMacroDesc    = FString::Printf(TEXT("\"Id\" argument in %s macro at %s:%d"), *GetToken(), *Context.Filename, Context.LineNumber);
                const FString KeyMacroDesc        = FString::Printf(TEXT("\"Key\" argument in %s macro at %s:%d"), *GetToken(), *Context.Filename, Context.LineNumber);
                const FString MetaDataIdMacroDesc = FString::Printf(TEXT("\"MetaDataId\" argument in %s macro at %s:%d"), *GetToken(), *Context.Filename, Context.LineNumber);
                const FString MetaDataMacroDesc   = FString::Printf(TEXT("\"MetaData\" argument in %s macro at %s:%d"), *GetToken(), *Context.Filename, Context.LineNumber);

                bool HasQuotes;
                if (PrepareArgument(TableId, true, TableIdMacroDesc, HasQuotes) && PrepareArgument(Key, true, KeyMacroDesc, HasQuotes) && PrepareArgument(MetaDataId, true, MetaDataIdMacroDesc, HasQuotes) && PrepareArgument(MetaData, true, MetaDataMacroDesc, HasQuotes))
                {
                    const FName TableIdName    = *TableId;
                    const FName MetaDataIdName = *MetaDataId;

                    if (TableIdName.IsNone() || Key.IsEmpty() || MetaDataIdName.IsNone())
                    {
                        UE_LOG(LogGatherTextFromSourceCommandlet, Warning, TEXT("%s macro at %s:%d has an empty identifier and cannot be gathered."), *GetToken(), *Context.Filename, Context.LineNumber);
                    }
                    else if (!MetaData.IsEmpty())
                    {
                        Context.AddStringTableEntryMetaData(TableIdName, Key, MetaDataIdName, MetaData);
                    }
                }
            }
        }
    }
}

void UGatherTextFromSourceCommandlet::FIniNamespaceDescriptor::TryParse(const FString& Text, FSourceFileParseContext& Context) const
{
    // Attempt to parse something of the format
    // [<config section name>]
    if (!Context.ExcludedRegion)
    {
        if (Context.LineText[0] == '[')
        {
            int32 ClosingBracket;
            if (Text.FindChar(']', ClosingBracket) && ClosingBracket > 1)
            {
                Context.Namespace             = Text.Mid(1, ClosingBracket - 1);
                Context.EndParsingCurrentLine = true;
            }
        }
    }
}

#undef LOC_DEFINE_REGION