Runtime content downloading

Co-authored-by: Oleg Petruny <oleg.petruny@gmail.com>
Co-committed-by: Oleg Petruny <oleg.petruny@gmail.com>

.gitattributes

@@ -1,3 +1,4 @@
 ReleaseBuilds/** filter=lfs diff=lfs merge=lfs -text
 Images/** filter=lfs diff=lfs merge=lfs -text
 Fonts/** filter=lfs diff=lfs merge=lfs -text
+Audio/** filter=lfs diff=lfs merge=lfs -text

UnrealProject/Lost_Edge/Source/Lost_Edge/Lost_Edge.Build.cs

@@ -9,7 +9,7 @@ public class Lost_Edge : ModuleRules {
         PublicDependencyModuleNames.AddRange(new string[] { "Core", "CoreUObject", "Engine", "InputCore", "OpenCV" });
 
         PrivateDependencyModuleNames.AddRange(new string[] { "EnhancedInput", "UMG", "RHI", "RenderCore", "Lost_EdgeShaders", "PakFile", //"TextureCompressor",
-            "LevelSequence", "MovieScene" }); // "Slate", "SlateCore"
+            "LevelSequence", "MovieScene", "HTTP", "Json" }); // "Slate", "SlateCore"
 
         // UE_LOG(LogTemp, Log, TEXT("capture: %s"), (capture ? TEXT("true") : TEXT("false")));
         // GEngine->AddOnScreenDebugMessage(-1, 5.0f, FColor::Yellow, TEXT("1"));

UnrealProject/Lost_Edge/Source/Lost_Edge/Private/CommonFunctions.h

@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include "CoreMinimal.h"
 #include "Kismet/BlueprintFunctionLibrary.h"
 
 #include "CommonFunctions.generated.h"
@@ -37,6 +38,12 @@ public:
 
     static FWorldDilationChangedDelegate& GetWorldDilationChangedDelegate();
 
+
+    template<typename T>
+    static TArray<T> ArrayDiff(const TArray<T>& a, const TArray<T>& b);
+    template<typename T>
+    static void ArrayRemoveFirstFromEnd(TArray<T>& array, T element);
+
 private:
     static void SlowMotionTick();
 
@@ -48,3 +55,37 @@ namespace SlowMotion
     constexpr float slowDilation = 0.2f;
     constexpr float interpolationSpeed = 0.1f;
 }
+
+template<typename T>
+inline TArray<T> UCommonFunctions::ArrayDiff(const TArray<T>& a, const TArray<T>& b)
+{
+    TSet<T> result;
+
+    if(a.Num() == 0)
+        return b;
+    else if(b.Num() == 0)
+        return a;
+
+    for(auto& i : a)
+        if(!b.Contains(i))
+            result.Add(i);
+
+    for(auto& i : b)
+        if(!a.Contains(i))
+            result.Add(i);
+
+    return result.Array();
+}
+
+template<typename T>
+inline void UCommonFunctions::ArrayRemoveFirstFromEnd(TArray<T>& array, T element)
+{
+    for(int32 i = array.Num() - 1; i > -1; --i)
+    {
+        if(array[i] == element)
+        {
+            array.RemoveAt(i);
+            return;
+        }
+    }
+}

UnrealProject/Lost_Edge/Source/Lost_Edge/Private/ContentLoader.cpp

@@ -3,7 +3,26 @@
 
 #include "ContentLoader.h"
 
+#include "CommonFunctions.h"
 #include "IPlatformFilePak.h"
+#include "Misc/FileHelper.h"
+
+#include <regex>
+#include <string>
+#include <vector>
+
+namespace
+{
+    constexpr auto httpGet = TEXT("GET");
+
+    constexpr int32 httpResponseOK = 200;
+
+    constexpr auto httpContentTypeHtml = TEXT("text/html");
+    constexpr auto httpContentTypeJson = TEXT("application/json");
+
+    constexpr auto pakIndexUrl = TEXT("https://pixelyfier.com/lost_edge/content/");
+    const auto pakFileSavePath = FString::Printf(TEXT("%sDownloaded/"), *FPaths::ProjectDir());
+}
 
 void UContentLoader::BeginDestroy()
 {
@@ -18,9 +37,129 @@ void UContentLoader::BeginDestroy()
     UObject::BeginDestroy();
 }
 
+void UContentLoader::HttpGet(const FString& url, FHttpRequestCompleteDelegate requestCompleteCallback)
+{
+    TSharedRef<IHttpRequest> request = FHttpModule::Get().CreateRequest();
+    request->SetURL(url);
+    request->SetVerb(httpGet);
+    request->OnProcessRequestComplete() = std::move(requestCompleteCallback);
+    request->ProcessRequest();
+}
+
+TArray<FString> UContentLoader::ParseDirectoryIndex(const TArray<uint8>& content, const FString& contentType)
+{
+    TArray<FString> result;
+
+    if(contentType == httpContentTypeHtml)
+    {
+        std::string contentHtml{ reinterpret_cast<const char*>(content.GetData()), static_cast<size_t>(content.Num()) };
+        std::regex hrefRegex(R"(<a href="((?!\.)[^"]+))");
+        std::smatch match;
+        auto it = contentHtml.cbegin();
+        auto end = contentHtml.cend();
+        while(std::regex_search(it, end, match, hrefRegex))
+        {
+            if(match.size() > 1)
+            {
+                result.Add(UTF8_TO_TCHAR(match[1].str().c_str()));
+            }
+            it = match.suffix().first;
+        }
+    }
+    else if(contentType == httpContentTypeJson)
+    {
+        // TODO
+    }
+
+    return result;
+}
+
+FString UContentLoader::SelectContentByMethod(const TArray<FString>& content, const EContentDownloadMethod method)
+{
+    switch(method)
+    {
+        case EContentDownloadMethod::NonRepeatRandom:
+        {
+            auto filteredContent = UCommonFunctions::ArrayDiff(content, downloadedContent);
+            if(filteredContent.Num() == 0 && content.Num() != 0)
+            {
+                downloadedContent.Reset();
+                filteredContent = UCommonFunctions::ArrayDiff(content, downloadedContent);
+            }
+            return filteredContent[FMath::RandRange(0, filteredContent.Num() - 1)];
+        }
+        case EContentDownloadMethod::RatingOftenRandom:
+            // TODO
+        case EContentDownloadMethod::ClearRandom:
+        default:
+            return content[FMath::RandRange(0, content.Num() - 1)];
+    }
+}
+
+void UContentLoader::DownloadPak(FContentDownloadedCallback downloadedCallback, const EContentDownloadMethod method)
+{
+    UE_LOG(LogTemp, Log, TEXT("ContentLoader: Requesting pak http index at %s."), pakIndexUrl);
+
+    FHttpRequestCompleteDelegate indexRequestCallback;
+    indexRequestCallback.BindLambda([=, this](FHttpRequestPtr request, FHttpResponsePtr response, bool successful)
+        {
+            if(!successful || response->GetResponseCode() != httpResponseOK)
+            {
+                int32 responseCode = response.IsValid() ? response->GetResponseCode() : -1;
+                UE_LOG(LogTemp, Log, TEXT("ContentLoader: Requesting pak http index failed with code: %d (-1 if response is null)."), responseCode);
+                return;
+            }
+
+            TArray<FString> availablePaks = ParseDirectoryIndex(response->GetContent(), response->GetContentType());
+            if(availablePaks.Num() == 0)
+            {
+                UE_LOG(LogTemp, Log, TEXT("ContentLoader: Pak http index is empty."));
+                return;
+            }
+
+            FString pakToDownload = SelectContentByMethod(availablePaks, method);
+            downloadedContent.Add(pakToDownload);
+            UE_LOG(LogTemp, Log, TEXT("ContentLoader: Pak selected for loading: %s."), *pakToDownload);
+
+            auto savePath = FString::Printf(TEXT("%s%s"), *pakFileSavePath, *pakToDownload);
+            if(FPlatformFileManager::Get().GetPlatformFile().FileExists(*savePath))
+            {
+                UE_LOG(LogTemp, Log, TEXT("ContentLoader: Selected pak already exists... skipping download."));
+                downloadedCallback.ExecuteIfBound(savePath);
+                return;
+            }
+
+            FHttpRequestCompleteDelegate pakDownloadCallback;
+            pakDownloadCallback.BindLambda([=, this](FHttpRequestPtr request, FHttpResponsePtr response, bool successful)
+                {
+                    if(!successful || response->GetResponseCode() != httpResponseOK)
+                    {
+                        UCommonFunctions::ArrayRemoveFirstFromEnd(downloadedContent, pakToDownload);
+                        int32 responseCode = response.IsValid() ? response->GetResponseCode() : -1;
+                        UE_LOG(LogTemp, Log, TEXT("ContentLoader: Pak downloading failed with code: %d (-1 if response is null)."), responseCode);
+                        return;
+                    }
+
+                    if(!FFileHelper::SaveArrayToFile(response->GetContent(), *savePath))
+                    {
+                        UE_LOG(LogTemp, Log, TEXT("ContentLoader: Saving pak file %s failed."), *savePath);
+                        return;
+                    }
+                    UE_LOG(LogTemp, Log, TEXT("ContentLoader: Pak file saved as %s."), *savePath);
+
+                    downloadedCallback.ExecuteIfBound(savePath);
+                });
+
+            UE_LOG(LogTemp, Log, TEXT("ContentLoader: Starting to download pak %s from index."), *pakToDownload);
+            HttpGet(FString::Printf(TEXT("%s%s"), pakIndexUrl, *pakToDownload), pakDownloadCallback);
+        });
+
+    HttpGet(pakIndexUrl, indexRequestCallback);
+}
+
 UClass* UContentLoader::LoadPak(const FString& pakFilePath)
 {
-    UE_LOG(LogTemp, Log, TEXT("ContentLoader: Starting to mount pak %s"), *pakFilePath);
+    UE_LOG(LogTemp, Log, TEXT("ContentLoader: Starting to mount pak %s."), *pakFilePath);
 
     FPakPlatformFile* pakFileMgr = (FPakPlatformFile*)(FPlatformFileManager::Get().FindPlatformFile(TEXT("PakFile")));
     if(!pakFileMgr)
@@ -104,13 +243,13 @@ bool UContentLoader::UnloadPak(const FString& pakFilePath)
     FPakPlatformFile* pakFileMgr = (FPakPlatformFile*)(FPlatformFileManager::Get().FindPlatformFile(TEXT("PakFile")));
     if(!pakFileMgr)
     {
-        UE_LOG(LogTemp, Warning, TEXT("Unable to get PakPlatformFile for pak file (Unmount): %s"), *pakFilePath);
+        UE_LOG(LogTemp, Warning, TEXT("ContentLoader: Unable to get PakPlatformFile for pak file (Unmount): %s."), *pakFilePath);
         return false;
     }
 
     if(!pakFileMgr->Unmount(*pakFilePath))
     {
-        UE_LOG(LogTemp, Warning, TEXT("Unable to unmount pak: %s"), *pakFilePath);
+        UE_LOG(LogTemp, Warning, TEXT("ContentLoader: Unable to unmount pak: %s."), *pakFilePath);
         return false;
     }
 

UnrealProject/Lost_Edge/Source/Lost_Edge/Private/ContentLoader.h

@@ -2,10 +2,21 @@
 
 #pragma once
 
+#include "Http.h"
 #include "UObject/Object.h"
 
 #include "ContentLoader.generated.h"
 
+UENUM(BlueprintType)
+enum class EContentDownloadMethod : uint8
+{
+    ClearRandom = 0,
+    NonRepeatRandom,
+    RatingOftenRandom
+};
+
+DECLARE_DYNAMIC_DELEGATE_OneParam(FContentDownloadedCallback, FString, pakFilePath);
+
 UCLASS(BlueprintType)
 class UContentLoader : public UObject
 {
@@ -13,15 +24,23 @@ class UContentLoader : public UObject
 
 public:
     UFUNCTION(BlueprintCallable, Category = "ContentLoader")
-    UClass* LoadPak(const FString& pakFilePath);
+    void DownloadPak(FContentDownloadedCallback downloadedCallback, const EContentDownloadMethod method = EContentDownloadMethod::NonRepeatRandom);
 
-protected:
+    UFUNCTION(BlueprintCallable, Category = "ContentLoader")
-    virtual void BeginDestroy() override;
+    UClass* LoadPak(const FString& pakFilePath);
 
     UFUNCTION(BlueprintCallable, Category = "ContentLoader")
     bool UnloadPak(const FString& pakFilePath);
 
 protected:
+    virtual void BeginDestroy() override;
+
+    void HttpGet(const FString& url, FHttpRequestCompleteDelegate requestCompleteCallback);
+    TArray<FString> ParseDirectoryIndex(const TArray<uint8>& content, const FString& contentType);
+    FString SelectContentByMethod(const TArray<FString>& content, const EContentDownloadMethod method);
+
     FString GetPakMountContent(const FString& pakFilePath, TArray<FString>* content = nullptr);
     UClass* GetPakClass(const FString& pakContentPath);
+
+    TArray<FString> downloadedContent;
 };

UnrealProject/Lost_Edge/Source/Lost_Edge/Private/DialogueManager.cpp

@@ -147,7 +147,7 @@ void UDialogueManager::PlayNextDialogue()
     {
         FDialogueEndCallback callback;
         _endCallbacks.Dequeue(callback);
-        callback.Execute();
+        callback.ExecuteIfBound();
         _nextDialogues.Pop();
         return;
     }