Compare commits

...

14 Commits

Author SHA1 Message Date
507fb88756 Merge pull request 'FIX - Delegate could be in header only' (#52) from fix-physic-post-step-delegate into main
Reviewed-on: #52
2025-04-30 14:43:26 +00:00
Jb win
7148f14b0d FIX - Delegate could be in header only 2025-04-30 21:40:45 +07:00
7d383871b8 Merge pull request 'Patch MuJoCo plugin' (#50) from ft-pre-post-substep-delegates into main
Reviewed-on: #50
Reviewed-by: evanvyktori <evan@luckyrobots.com>
2025-04-30 14:22:03 +00:00
Jb win
59377bdd4f FIX - SO100 works in kitchen level
+ Rotation values are better
+ changed Kp value back to 300 like Pitch
+ Added Contact exclusion rule
+ Fixed Range for body and control
+ Restore proper delta values
2025-04-30 13:24:03 +00:00
Jb win
6e35f6bcff FIX - Typo 2025-04-30 13:24:03 +00:00
Jb win
5ee0e40e6f FIX - InitializeMujoco is now a wrapper to the real SceneInitialization
+ I don't want to commit 10 blueprints because we can't put empty map as default values in BP nodes
2025-04-30 13:24:03 +00:00
Jb win
2207cfbba0 FT - Contact Exclusion list for XML scene creation
+ This feature was missing from the plugin while being necessary for robots to work
+ e.g. SO100 requires the main body and the first arm to have contact exclusion, I guess the piece are too tight in the model?
+ This is not clean, but making it right requires to have a better understanding of how the MuJoCo actor works - problem for future self
2025-04-30 13:24:03 +00:00
Jb win
a9cd14ae7f FT - Access low level mujoco data from anywhere
+ be careful, this is not const!
2025-04-30 13:24:03 +00:00
Jb win
d311284050 FT - PostStep Update as a Delegate
+ Can be used to finely control actuators between mj_steps
2025-04-30 13:24:03 +00:00
568d67d04f Merge pull request 'Create RenderTarget Utilitiy Library for Capturing Images to Texture2D Object' (#47) from Ancient23/LuckyWorldV2:main into main
Reviewed-on: #47
Reviewed-by: evanvyktori <evan@luckyrobots.com>
2025-04-29 23:49:17 +00:00
Filip Iliescu
a3347489bc Add RenderCore to plugin 2025-04-29 16:44:32 -07:00
Filip Iliescu
5cc5b232cb Clean up and document code 2025-04-29 16:00:18 -07:00
Filip Iliescu
8dc892b9fa Scaffold render target library 2025-04-29 15:40:38 -07:00
22e2b0a419 Merge pull request 'Noah_DataTransfer_1.0' (#39) from Noah_DataTransfer_1.0 into main
Reviewed-on: #39
Reviewed-by: evanvyktori <evan@luckyrobots.com>
2025-04-29 14:48:56 +00:00
12 changed files with 372 additions and 32 deletions

View File

@ -111,7 +111,12 @@ template <typename ComponentType> void AMujocoVolumeActor::AssignComponentsToArr
void AMujocoVolumeActor::InitializeMujoco()
{
if (!Options)
InitializeMujocoScene_WithContactExclusion(TMap<FString, FString>{});
}
void AMujocoVolumeActor::InitializeMujocoScene_WithContactExclusion(const TMap<FString, FString>& ContactExclusion)
{
if (!Options)
{
return;
}
@ -149,7 +154,7 @@ void AMujocoVolumeActor::InitializeMujoco()
}
TUniquePtr<FMujocoXmlGenerator> Generator = MakeUnique<FMujocoXmlGenerator>();
TUniquePtr<tinyxml2::XMLDocument> Doc = Generator->GenerateMujocoXml(Options, Objects, ExportFilename);
TUniquePtr<tinyxml2::XMLDocument> Doc = Generator->GenerateMujocoXml(Options, Objects, ExportFilename, ContactExclusion);
FString XmlString;
tinyxml2::XMLPrinter Printer;
@ -164,6 +169,8 @@ void AMujocoVolumeActor::InitializeMujoco()
return;
}
// TODO Here we should check for mujoco dll if we are on windows and LOG an error if there is not + gently quit the game
std::array<char, 1024> ErrMsg{};
MujocoModel = MakeMujocoModelPtr(mj_loadXML(TCHAR_TO_ANSI(*ExportFilename), nullptr, ErrMsg.data(), ErrMsg.size()));
if (!MujocoModel)
@ -196,6 +203,12 @@ void AMujocoVolumeActor::InitializeMujoco()
}
}
mjData_& AMujocoVolumeActor::GetMujocoData() const
{
check(MujocoData.IsValid());
return *MujocoData.Get();
}
void AMujocoVolumeActor::SetActuatorValue(const FString& ActuatorName, double Value)
{
if (MujocoModel)
@ -391,11 +404,13 @@ void AMujocoVolumeActor::Tick(float DeltaTime)
{
if (MujocoData)
{
mj_step(MujocoModel.Get(), MujocoData.Get());
for (int32 Frame = 0; Frame < FrameSkip; ++Frame)
// Default SimStepsPerGameFrame = 16 x 1ms for roughly 60FPS
// If ticks are strictly 16.666 ms then simulation will lag behind UE clock
// Anyway, we need a proper synchronisation between UE and Sim times - probably using TempoTime?
for (int32 SimStep = 0; SimStep < SimStepsPerGameFrame; ++SimStep)
{
mj_step(MujocoModel.Get(), MujocoData.Get());
PostPhysicStepDelegate.ExecuteIfBound(MujocoData->time);
}
for (int32 i = 1; i < BodyComponents.Num(); ++i)

View File

@ -1255,7 +1255,11 @@ bool FMujocoXmlGenerator::ParseActorAsset(AActor* Actor, tinyxml2::XMLElement* R
return true;
}
TUniquePtr<tinyxml2::XMLDocument> FMujocoXmlGenerator::GenerateMujocoXml(const TObjectPtr<UMujocoExportOptions>& ExportOptions, const TArray<UObject*>& Objects, const FString& ExportFilename)
TUniquePtr<tinyxml2::XMLDocument> FMujocoXmlGenerator::GenerateMujocoXml(
const TObjectPtr<UMujocoExportOptions>& ExportOptions,
const TArray<UObject*>& Objects,
const FString& ExportFilename,
TMap<FString, FString> ContactExclusion)
{
TUniquePtr<tinyxml2::XMLDocument> Doc = MakeUnique<tinyxml2::XMLDocument>();
Doc->InsertFirstChild(Doc->NewDeclaration("xml version=\"1.0\" encoding=\"utf-8\""));
@ -1268,6 +1272,30 @@ TUniquePtr<tinyxml2::XMLDocument> FMujocoXmlGenerator::GenerateMujocoXml(const T
Root->InsertEndChild(Doc->NewElement("equality"));
Root->InsertEndChild(Doc->NewElement("tendon"));
Root->InsertEndChild(Doc->NewElement("actuator"));
// TODO Refacto using property from the MujocoActor aka the robot
// TODO Maybe using a ParentObject as parameter which holds that information?
// TODO Need actor refactoring anyway - Merge Pawn and Mujoco Actor?
if (!ContactExclusion.IsEmpty())
{
Root->InsertEndChild(Doc->NewElement("contact"));
for (auto& [Body1, Body2] : ContactExclusion)
{
// Don't add empty strings to the map please!
if (Body1.IsEmpty() || Body2.IsEmpty()) continue;
// Create the contact exclusion - formatted as (SO100 example)
// <exclude body1="Body_Base" body2="Body_Rotation_Pitch"/>
const auto Exclude = Doc->NewElement("exclude");
Exclude->SetAttribute("body1", TCHAR_TO_ANSI(*Body1));
Exclude->SetAttribute("body2", TCHAR_TO_ANSI(*Body2));
// Add exclusion to the contact tag
Root->FirstChildElement("contact")->InsertEndChild(Exclude);
}
}
if (ExportOptions->bAddSkyBox)
{
tinyxml2::XMLElement* TextureElement = AssetRoot->GetDocument()->NewElement("texture");

View File

@ -21,11 +21,44 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnMujocoCompileBegin);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnMujocoCompileError, FString, Error);
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnMujocoCompileSuccess);
// To be called after mj_step
DECLARE_DELEGATE_OneParam(FPostPhysicUpdate, float);
UCLASS(Blueprintable, BlueprintType)
class LUCKYMUJOCO_API AMujocoVolumeActor : public AActor
{
GENERATED_BODY()
public:
AMujocoVolumeActor();
/**
* Initialize the sim scene in headless mujoco
*/
UFUNCTION(BlueprintCallable, Category = "Mujoco")
void InitializeMujoco();
/**
* Initialize the sim scene in headless mujoco with a list of contact exclusion
* @param ContactExclusion a list of pairs that should be patched in the xml file for contact exclusion (no friction, no collision)
* TODO Can't use a default empty map as parameter in blueprints? We shouldn't need to have 2 functions
* TODO ContactExclusion should be stored as a property of MujocoActor and not passed in the scene init
* TODO This require to cast the Actor in addition to the components list
*/
UFUNCTION(BlueprintCallable, Category = "Mujoco")
void InitializeMujocoScene_WithContactExclusion(const TMap<FString, FString>& ContactExclusion);
protected:
virtual void BeginPlay() override;
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
virtual void PostRegisterAllComponents() override;
virtual void Tick(float DeltaTime) override;
virtual void PostInitializeComponents() override;
private:
TMujocoModelPtr MujocoModel;
TMujocoDataPtr MujocoData;
@ -50,16 +83,11 @@ class LUCKYMUJOCO_API AMujocoVolumeActor : public AActor
UPROPERTY(Transient, VisibleAnywhere, Category = "Mujoco | Debug")
TArray<TSoftObjectPtr<UMujocoTendonComponent>> TendonComponents;
UFUNCTION(BlueprintCallable, Category = "Mujoco")
void InitializeMujoco();
template <typename ComponentType> void AssignComponentsToArray(UWorld* World, TArray<TSoftObjectPtr<ComponentType>>& ComponentArray);
public:
AMujocoVolumeActor();
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mujoco | Simulation", meta = (Min = 0, Max = 100, ClampMin = 0, ClampMax = 100))
int32 FrameSkip = 0;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Mujoco | Simulation", meta = (Min = 1, Max = 100, ClampMin = 0, ClampMax = 100))
int32 SimStepsPerGameFrame = 16;
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite, Category = "Mujoco")
TObjectPtr<UBillboardComponent> SpriteComponent;
@ -76,6 +104,33 @@ public:
UPROPERTY(BlueprintAssignable, Category = "Mujoco | Events")
FOnMujocoCompileSuccess OnMujocoCompileSuccess;
/**
* Access MuJoCo scene options and data (e.g. Simulation time) - This is mutable, be careful
* @return mjData_ - Full access to mujoco scene options and data
*/
mjData_& GetMujocoData() const;
// ---------------------------
// ------- POST UPDATE -------
// ---------------------------
private:
FPostPhysicUpdate PostPhysicStepDelegate;
public:
/**
* Register a delegate to be executed after mj_step, useful to fine control actuators
* @param Object - The Class to call
* @param Func - The Function to call - takes a float as parameter which is the current simulation timestamp
*/
template<typename UserClass>
void BindPostPhysicStepDelegate(UserClass* Object, void (UserClass::*Func)(float))
{
PostPhysicStepDelegate.BindUObject(Object, Func);
}
// -------------------------
// ------- ACTUATORS -------
// -------------------------
UFUNCTION(BlueprintCallable, Category = "Mujoco")
void SetActuatorValue(const FString& ActuatorName, double Value);
@ -94,6 +149,9 @@ public:
UFUNCTION(BlueprintCallable, BlueprintPure, Category = "Mujoco")
FVector2D GetActuatorRangeByIndex(int32 ActuatorIndex) const;
// ----------------------
// ------- JOINTS -------
// ----------------------
UFUNCTION(BlueprintCallable, Category = "Mujoco")
void SetJointValue(const FString& JointName, double Value);
@ -105,16 +163,4 @@ public:
UFUNCTION(BlueprintCallable, BlueprintPure, Category = "Mujoco")
double GetJointValueByIndex(int32 JointIndex) const;
virtual void PostRegisterAllComponents() override;
virtual void Tick(float DeltaTime) override;
virtual void PostInitializeComponents() override;
protected:
virtual void BeginPlay() override;
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
};

View File

@ -307,13 +307,13 @@ struct LUCKYMUJOCO_API FMujocoOptions
{
GENERATED_BODY()
/** Override TimeStep setting. When enabled, the default value (0.002) is applied. */
/** Override TimeStep setting. When enabled, the default value (0.001) is applied. */
UPROPERTY(config, EditAnywhere, BlueprintReadWrite, Category = "Mujoco", meta = (InlineEditConditionToggle))
bool bOverrideTimeStep = false;
/** Simulation time step in seconds. */
UPROPERTY(config, EditAnywhere, BlueprintReadWrite, Category = "Mujoco", meta = (Attribute = "timestep", EditCondition = "bOverrideTimeStep", ClampMin = 0.0001, ClampMax = 0.01, UIMin = 0.0001, UIMax = 0.01))
float TimeStep = 0.002f;
float TimeStep = 0.001f; // Default to 1ms
/** Override API Rate setting. */
UPROPERTY(config, EditAnywhere, BlueprintReadWrite, Category = "Mujoco", meta = (InlineEditConditionToggle))

View File

@ -35,6 +35,19 @@ class LUCKYMUJOCO_API FMujocoXmlGenerator
TMap<FString, TSoftObjectPtr<UObject>> ObjectMap;
public:
TUniquePtr<tinyxml2::XMLDocument> GenerateMujocoXml(const TObjectPtr<UMujocoExportOptions>& ExportOptions, const TArray<UObject*>& Objects, const FString& ExportFilename);
public:
/**
* Generate the XML file used by the MuJoCo headless scene representing the physics simulation
* @param ExportOptions TODO - What they can be?
* @param Objects Actors and Components to be added to the scene
* @param ExportFilename pretty much that
* @param ContactExclusion a list of pairs that should be patched in the xml file for contact exclusion (no friction, no collision)
* TODO ContactExclusion should be stored as a property of MujocoActor and not passed in the scene init
*/
TUniquePtr<tinyxml2::XMLDocument> GenerateMujocoXml(
const TObjectPtr<UMujocoExportOptions>& ExportOptions,
const TArray<UObject*>& Objects,
const FString& ExportFilename,
TMap<FString, FString> ContactExclusion);
};

View File

@ -141,8 +141,8 @@ public:
CancelButton->SetEnabled(false);
RequestDestroyWindow();
TUniquePtr<FMujocoXmlGenerator> Generator = MakeUnique<FMujocoXmlGenerator>();
TUniquePtr<tinyxml2::XMLDocument> Doc = Generator->GenerateMujocoXml(ExportOptions, Objects, ExportFilename);
const TUniquePtr<FMujocoXmlGenerator> Generator = MakeUnique<FMujocoXmlGenerator>();
const TUniquePtr<tinyxml2::XMLDocument> Doc = Generator->GenerateMujocoXml(ExportOptions, Objects, ExportFilename, TMap<FString, FString>{});
FString XmlString;
tinyxml2::XMLPrinter Printer;

View File

@ -19,7 +19,8 @@ public class LuckyWorldV2 : ModuleRules
"SocketIOClient",
"VaRest",
"SIOJson",
"NavigationSystem"
"NavigationSystem",
"RenderCore"
});
PrivateDependencyModuleNames.AddRange(new string[] { });

View File

@ -0,0 +1,180 @@
// Fill out your copyright notice in the Description page of Project Settings.
#include "LRRenderUtilLibrary.h"
#include "Components/SceneCaptureComponent2D.h"
#include "Engine/TextureRenderTarget2D.h"
#include "Kismet/KismetRenderingLibrary.h"
#include "GameFramework/Actor.h"
#include "Engine/World.h"
void ULRRenderUtilLibrary::SetupSceneCaptureForMesh(
USceneCaptureComponent2D* SceneCapture,
UStaticMeshComponent* MeshComp,
TEnumAsByte<enum ESceneCaptureSource> CaptureSource)
{
if (!SceneCapture || !MeshComp)
{
UE_LOG(LogTemp, Warning, TEXT("SetupSceneCaptureForMesh: Invalid SceneCapture or MeshComp"));
return;
}
if (!MeshComp->IsValidLowLevel())
{
UE_LOG(LogTemp, Warning, TEXT("SetupSceneCaptureForMesh: MeshComp is not valid"));
return;
}
// Set the scene capture to focus on the mesh
SceneCapture->ShowOnlyComponent(MeshComp);
SceneCapture->CaptureSource = CaptureSource;
SceneCapture->bCaptureEveryFrame = false;
SceneCapture->bCaptureOnMovement = false;
SceneCapture->FOVAngle = 90.f;
// Set a solid background color
SceneCapture->CompositeMode = ESceneCaptureCompositeMode::SCCM_Overwrite;
// Ensure we have a valid texture target
if (SceneCapture->TextureTarget)
{
SceneCapture->TextureTarget->ClearColor = FLinearColor::Transparent;
}
else
{
UE_LOG(LogTemp, Warning, TEXT("SetupSceneCaptureForMesh: TextureTarget is null"));
return;
}
// Adjust the transform to frame the mesh
FVector MeshBounds = MeshComp->Bounds.BoxExtent;
FVector MeshOrigin = MeshComp->Bounds.Origin;
// Calculate the camera distance based on mesh size
float MaxExtent = FMath::Max3(MeshBounds.X, MeshBounds.Y, MeshBounds.Z);
float CameraDistance = MaxExtent * 2.5f; // Adjusted to ensure mesh fits in view
// Calculate the camera location - position from front of mesh (+X axis)
FVector CameraLocation = MeshOrigin + FVector(CameraDistance, 0.f, 0.f);
// Calculate the camera rotation to look at mesh center
FRotator CameraRotation = (MeshOrigin - CameraLocation).Rotation();
SceneCapture->SetWorldLocationAndRotation(CameraLocation, CameraRotation);
// Register component to make sure it will render
if (!SceneCapture->IsRegistered())
{
SceneCapture->RegisterComponent();
}
}
UTextureRenderTarget2D* ULRRenderUtilLibrary::CaptureMeshToRenderTarget(
UStaticMeshComponent* MeshComp,
const FVector2D& ImageSize,
UObject* WorldContextObject,
TEnumAsByte<enum ESceneCaptureSource> CaptureSource)
{
if (!MeshComp || !WorldContextObject || !WorldContextObject->GetWorld())
{
UE_LOG(LogTemp, Warning, TEXT("CaptureMeshToRenderTarget: Invalid parameters"));
return nullptr;
}
// Create a render target with proper size
UTextureRenderTarget2D* RenderTarget = UKismetRenderingLibrary::CreateRenderTarget2D(
WorldContextObject,
FMath::Max(32, static_cast<int32>(ImageSize.X)),
FMath::Max(32, static_cast<int32>(ImageSize.Y)),
ETextureRenderTargetFormat::RTF_RGBA8);
if (!RenderTarget)
{
UE_LOG(LogTemp, Warning, TEXT("CaptureMeshToRenderTarget: Failed to create render target"));
return nullptr;
}
// Create a temporary actor to hold the scene capture component
AActor* TempActor = WorldContextObject->GetWorld()->SpawnActor<AActor>();
if (!TempActor)
{
UE_LOG(LogTemp, Warning, TEXT("CaptureMeshToRenderTarget: Failed to spawn temporary actor"));
return RenderTarget;
}
// Create a root component if it doesn't exist
if (!TempActor->GetRootComponent())
{
USceneComponent* RootComponent = NewObject<USceneComponent>(TempActor, TEXT("RootComponent"));
TempActor->SetRootComponent(RootComponent);
RootComponent->RegisterComponent();
}
// Create and set up scene capture component
USceneCaptureComponent2D* SceneCapture = NewObject<USceneCaptureComponent2D>(TempActor);
if (SceneCapture)
{
SceneCapture->RegisterComponent();
SceneCapture->AttachToComponent(TempActor->GetRootComponent(), FAttachmentTransformRules::KeepRelativeTransform);
SceneCapture->TextureTarget = RenderTarget;
// Setup the scene capture
SetupSceneCaptureForMesh(SceneCapture, MeshComp, CaptureSource);
// Clear the render target
UKismetRenderingLibrary::ClearRenderTarget2D(WorldContextObject, RenderTarget, FLinearColor::Transparent);
// Disable post-processing effects
SceneCapture->PostProcessSettings.bOverride_AutoExposureMethod = true;
SceneCapture->PostProcessSettings.AutoExposureMethod = EAutoExposureMethod::AEM_Manual;
SceneCapture->PostProcessSettings.bOverride_AutoExposureBias = true;
SceneCapture->PostProcessSettings.AutoExposureBias = 1.0f;
SceneCapture->PostProcessSettings.bOverride_BloomIntensity = true;
SceneCapture->PostProcessSettings.BloomIntensity = 0.0f;
// Trigger a one-time capture
SceneCapture->CaptureScene();
// Force the GPU to complete rendering
FlushRenderingCommands();
}
else
{
UE_LOG(LogTemp, Warning, TEXT("CaptureMeshToRenderTarget: Failed to create scene capture component"));
}
// Clean up the temporary actor
TempActor->Destroy();
return RenderTarget;
}
bool ULRRenderUtilLibrary::CaptureSceneNow(USceneCaptureComponent2D* SceneCapture)
{
if (!SceneCapture || !SceneCapture->IsValidLowLevel())
{
UE_LOG(LogTemp, Warning, TEXT("CaptureSceneNow: Invalid scene capture component"));
return false;
}
if (!SceneCapture->TextureTarget)
{
UE_LOG(LogTemp, Warning, TEXT("CaptureSceneNow: Scene capture has no texture target"));
return false;
}
// Ensure component is registered
if (!SceneCapture->IsRegistered())
{
SceneCapture->RegisterComponent();
}
// Trigger the capture
SceneCapture->CaptureScene();
// Force the GPU to complete rendering
FlushRenderingCommands();
return true;
}

View File

@ -0,0 +1,57 @@
// Fill out your copyright notice in the Description page of Project Settings.
#pragma once
#include "CoreMinimal.h"
#include "Kismet/BlueprintFunctionLibrary.h"
#include "LRRenderUtilLibrary.generated.h"
class USceneCaptureComponent2D;
class UTextureRenderTarget2D;
class UStaticMeshComponent;
/**
* Utility library for capturing and rendering objects in the Lucky Robots system
*/
UCLASS(BlueprintType, Blueprintable)
class LUCKYWORLDV2_API ULRRenderUtilLibrary : public UBlueprintFunctionLibrary
{
GENERATED_BODY()
public:
/**
* Captures a static mesh component to a render target texture
* @param MeshComp The static mesh component to capture
* @param ImageSize The size of the output image
* @param WorldContextObject The world context
* @param CaptureSource The type of data to capture (default is BaseColor)
* @return The rendered texture
*/
UFUNCTION(BlueprintCallable, Category = "Lucky Robots|Render Utilities")
static UTextureRenderTarget2D* CaptureMeshToRenderTarget(
UStaticMeshComponent* MeshComp,
const FVector2D& ImageSize,
UObject* WorldContextObject,
TEnumAsByte<enum ESceneCaptureSource> CaptureSource = ESceneCaptureSource::SCS_BaseColor);
/**
* Sets up a scene capture component to focus on a specific mesh
* @param SceneCapture The scene capture component to set up
* @param MeshComp The static mesh component to focus on
* @param CaptureSource The type of data to capture (default is BaseColor)
*/
UFUNCTION(BlueprintCallable, Category = "Lucky Robots|Render Utilities")
static void SetupSceneCaptureForMesh(
USceneCaptureComponent2D* SceneCapture,
UStaticMeshComponent* MeshComp,
TEnumAsByte<enum ESceneCaptureSource> CaptureSource = ESceneCaptureSource::SCS_BaseColor);
/**
* Captures the current view of a scene capture component to its render target
* @param SceneCapture The scene capture component to trigger
* @return Success status
*/
UFUNCTION(BlueprintCallable, Category = "Lucky Robots|Render Utilities")
static bool CaptureSceneNow(USceneCaptureComponent2D* SceneCapture);
};