Open3D (C++ API)  0.16.1
FilamentScene.h
Go to the documentation of this file.
1// ----------------------------------------------------------------------------
2// - Open3D: www.open3d.org -
3// ----------------------------------------------------------------------------
4// The MIT License (MIT)
5//
6// Copyright (c) 2018-2021 www.open3d.org
7//
8// Permission is hereby granted, free of charge, to any person obtaining a copy
9// of this software and associated documentation files (the "Software"), to deal
10// in the Software without restriction, including without limitation the rights
11// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12// copies of the Software, and to permit persons to whom the Software is
13// furnished to do so, subject to the following conditions:
14//
15// The above copyright notice and this permission notice shall be included in
16// all copies or substantial portions of the Software.
17//
18// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24// IN THE SOFTWARE.
25// ----------------------------------------------------------------------------
26
27#pragma once
28
29// 4068: Filament has some clang-specific vectorizing pragma's that MSVC flags
30// 4146: Filament's utils/algorithm.h utils::details::ctz() tries to negate
31// an unsigned int.
32// 4293: Filament's utils/algorithm.h utils::details::clz() does strange
33// things with MSVC. Somehow sizeof(unsigned int) > 4, but its size is
34// 32 so that x >> 32 gives a warning. (Or maybe the compiler can't
35// determine the if statement does not run.)
36// 4305: LightManager.h needs to specify some constants as floats
37#ifdef _MSC_VER
38#pragma warning(push)
39#pragma warning(disable : 4068 4146 4293 4305)
40#endif // _MSC_VER
41
42#include <filament/LightManager.h>
43#include <filament/RenderableManager.h>
44#include <utils/Entity.h>
45
46#ifdef _MSC_VER
47#pragma warning(pop)
48#endif // _MSC_VER
49
50#include <Eigen/Geometry>
51#include <unordered_map>
52#include <vector>
53
60
62namespace filament {
63class Box;
64class Engine;
65class IndirectLight;
66class Renderer;
67class Scene;
68class Skybox;
69class TransformManager;
70class VertexBuffer;
71} // namespace filament
73
74namespace open3d {
75namespace visualization {
76namespace rendering {
77
78class FilamentView;
79class GeometryBuffersBuilder;
80class Renderer;
81class View;
82
83// Contains renderable objects like geometry and lights
84// Can have multiple views
85class FilamentScene : public Scene {
86public:
87 using Transform = Eigen::Transform<float, 3, Eigen::Affine>;
88
89 FilamentScene(filament::Engine& engine,
90 FilamentResourceManager& resource_mgr,
91 Renderer& renderer);
93
94 Scene* Copy() override;
95
96 // NOTE: Temporarily needed to support old View interface for ImGUI
100 std::uint32_t h) override;
101
102 View* GetView(const ViewHandle& view_id) const override;
103 void SetViewActive(const ViewHandle& view_id, bool is_active) override;
104 void SetRenderOnce(const ViewHandle& view_id) override;
105 void RemoveView(const ViewHandle& view_id) override;
106
107 // Camera
108 void AddCamera(const std::string& camera_name,
109 std::shared_ptr<Camera> cam) override;
110 void RemoveCamera(const std::string& camera_name) override;
111 void SetActiveCamera(const std::string& camera_name) override;
112
113 // Scene geometry
114 bool AddGeometry(const std::string& object_name,
115 const geometry::Geometry3D& geometry,
116 const MaterialRecord& material,
117 const std::string& downsampled_name = "",
118 size_t downsample_threshold = SIZE_MAX) override;
119 bool AddGeometry(const std::string& object_name,
120 const t::geometry::Geometry& geometry,
121 const MaterialRecord& material,
122 const std::string& downsampled_name = "",
123 size_t downsample_threshold = SIZE_MAX) override;
124 bool AddGeometry(const std::string& object_name,
125 const TriangleMeshModel& model) override;
126 bool HasGeometry(const std::string& object_name) const override;
127 void UpdateGeometry(const std::string& object_name,
128 const t::geometry::PointCloud& point_cloud,
129 uint32_t update_flags) override;
130 void RemoveGeometry(const std::string& object_name) override;
131 void ShowGeometry(const std::string& object_name, bool show) override;
132 bool GeometryIsVisible(const std::string& object_name) override;
133 void SetGeometryTransform(const std::string& object_name,
134 const Transform& transform) override;
135 Transform GetGeometryTransform(const std::string& object_name) override;
137 const std::string& object_name) override;
138 void GeometryShadows(const std::string& object_name,
139 bool cast_shadows,
140 bool receive_shadows) override;
141 void SetGeometryCulling(const std::string& object_name,
142 bool enable) override;
143 void SetGeometryPriority(const std::string& object_name,
144 uint8_t priority) override;
145 void OverrideMaterial(const std::string& object_name,
146 const MaterialRecord& material) override;
147 void QueryGeometry(std::vector<std::string>& geometry) override;
148
149 void OverrideMaterialAll(const MaterialRecord& material,
150 bool shader_only = true) override;
151
152 // Lighting Environment
153 bool AddPointLight(const std::string& light_name,
154 const Eigen::Vector3f& color,
155 const Eigen::Vector3f& position,
156 float intensity,
157 float falloff,
158 bool cast_shadows) override;
159 bool AddSpotLight(const std::string& light_name,
160 const Eigen::Vector3f& color,
161 const Eigen::Vector3f& position,
162 const Eigen::Vector3f& direction,
163 float intensity,
164 float falloff,
165 float inner_cone_angle,
166 float outer_cone_angle,
167 bool cast_shadows) override;
168 bool AddDirectionalLight(const std::string& light_name,
169 const Eigen::Vector3f& color,
170 const Eigen::Vector3f& direction,
171 float intensity,
172 bool cast_shadows) override;
173 Light& GetLight(const std::string& light_name) override;
174 void RemoveLight(const std::string& light_name) override;
175 void UpdateLight(const std::string& light_name,
176 const Light& light) override;
177 void UpdateLightColor(const std::string& light_name,
178 const Eigen::Vector3f& color) override;
179 void UpdateLightPosition(const std::string& light_name,
180 const Eigen::Vector3f& position) override;
181 void UpdateLightDirection(const std::string& light_name,
182 const Eigen::Vector3f& direction) override;
183 void UpdateLightIntensity(const std::string& light_name,
184 float intensity) override;
185 void UpdateLightFalloff(const std::string& light_name,
186 float falloff) override;
187 void UpdateLightConeAngles(const std::string& light_name,
188 float inner_cone_angle,
189 float outer_cone_angle) override;
190 void EnableLightShadow(const std::string& light_name,
191 bool cast_shadows) override;
192
193 void SetSunLight(const Eigen::Vector3f& direction,
194 const Eigen::Vector3f& color,
195 float intensity) override;
196 void EnableSunLight(bool enable) override;
197 void EnableSunLightShadows(bool enable) override;
198 void SetSunLightColor(const Eigen::Vector3f& color) override;
199 Eigen::Vector3f GetSunLightColor() override;
200 void SetSunLightIntensity(float intensity) override;
201 float GetSunLightIntensity() override;
202 void SetSunLightDirection(const Eigen::Vector3f& direction) override;
203 Eigen::Vector3f GetSunLightDirection() override;
204 void SetSunAngularRadius(float radius) override;
205 void SetSunHaloSize(float size) override;
206 void SetSunHaloFalloff(float falloff) override;
207
208 bool SetIndirectLight(const std::string& ibl_name) override;
209 const std::string& GetIndirectLight() override;
210 void EnableIndirectLight(bool enable) override;
211 void SetIndirectLightIntensity(float intensity) override;
212 float GetIndirectLightIntensity() override;
213 void SetIndirectLightRotation(const Transform& rotation) override;
215 void ShowSkybox(bool show) override;
216 bool GetSkyboxVisible() const override;
217 void SetBackground(
218 const Eigen::Vector4f& color,
219 const std::shared_ptr<geometry::Image> image = nullptr) override;
220 void SetBackground(TextureHandle image) override;
221 void EnableGroundPlane(bool enable, GroundPlane plane) override;
222 void SetGroundPlaneColor(const Eigen::Vector4f& color) override;
223
224 void RenderToImage(std::function<void(std::shared_ptr<geometry::Image>)>
225 callback) override;
227 std::function<void(std::shared_ptr<geometry::Image>)> callback)
228 override;
229
230 void Draw(filament::Renderer& renderer);
231
232 // NOTE: This method is to work around Filament limitation when rendering to
233 // depth buffer. Materials with SSR require multiple passes which causes a
234 // crash with render to depth since we must disable multiple passes (i.e.,
235 // post-processing) in order to get back valid, un-modified depth values.
236 void HideRefractedMaterials(bool hide = true);
237
238 // NOTE: Can GetNativeScene be removed?
239 filament::Scene* GetNativeScene() const { return scene_; }
240
241private:
242 MaterialInstanceHandle AssignMaterialToFilamentGeometry(
243 filament::RenderableManager::Builder& builder,
244 const MaterialRecord& material);
245 enum BufferReuse { kNo, kYes };
246 bool CreateAndAddFilamentEntity(
247 const std::string& object_name,
248 GeometryBuffersBuilder& buffer_builder,
249 filament::Box& aabb,
252 const MaterialRecord& material,
253 BufferReuse reusing_vertex_buffer = BufferReuse::kNo);
254
255 filament::Engine& engine_;
256 FilamentResourceManager& resource_mgr_;
257 filament::Scene* scene_ = nullptr;
258
259 struct TextureMaps {
260 rendering::TextureHandle albedo_map =
262 rendering::TextureHandle normal_map =
264 rendering::TextureHandle ao_rough_metal_map =
266 rendering::TextureHandle reflectance_map =
268 rendering::TextureHandle clear_coat_map =
270 rendering::TextureHandle clear_coat_roughness_map =
272 rendering::TextureHandle anisotropy_map =
274 rendering::TextureHandle gradient_texture =
276 };
277
278 struct GeometryMaterialInstance {
279 TextureMaps maps;
280 MaterialRecord properties;
281 MaterialInstanceHandle mat_instance;
282 };
283
284 struct RenderableGeometry {
285 std::string name;
286 bool visible = true;
287 bool was_hidden_before_picking = false;
288 bool cast_shadows = true;
289 bool receive_shadows = true;
290 bool culling_enabled = true;
291 int priority = -1; // default priority
292
293 GeometryMaterialInstance mat;
294
295 // Filament resources
296 utils::Entity filament_entity;
297 filament::RenderableManager::PrimitiveType primitive_type;
300 void ReleaseResources(filament::Engine& engine,
301 FilamentResourceManager& manager);
302 };
303
304 struct LightEntity {
305 bool enabled = true;
306 utils::Entity filament_entity;
307 };
308
309 // NOTE: ViewContainer and views_ are temporary
310 struct ViewContainer {
311 std::unique_ptr<FilamentView> view;
312 bool is_active = true;
313 int render_count = -1;
314 };
315 std::unordered_map<REHandle_abstract, ViewContainer> views_;
316
317 std::vector<RenderableGeometry*> GetGeometry(const std::string& object_name,
318 bool warn_if_not_found = true);
319 bool GeometryIsModel(const std::string& object_name) const;
320 LightEntity* GetLightInternal(const std::string& light_name,
321 bool warn_if_not_found = true);
322 void OverrideMaterialInternal(RenderableGeometry* geom,
323 const MaterialRecord& material,
324 bool shader_only = false);
325 void UpdateMaterialProperties(RenderableGeometry& geom);
326 void UpdateDefaultLit(GeometryMaterialInstance& geom_mi);
327 void UpdateDefaultLitSSR(GeometryMaterialInstance& geom_mi);
328 void UpdateDefaultUnlit(GeometryMaterialInstance& geom_mi);
329 void UpdateNormalShader(GeometryMaterialInstance& geom_mi);
330 void UpdateDepthShader(GeometryMaterialInstance& geom_mi);
331 void UpdateDepthValueShader(GeometryMaterialInstance& geom_mi);
332 void UpdateGradientShader(GeometryMaterialInstance& geom_mi);
333 void UpdateSolidColorShader(GeometryMaterialInstance& geom_mi);
334 void UpdateBackgroundShader(GeometryMaterialInstance& geom_mi);
335 void UpdateGroundPlaneShader(GeometryMaterialInstance& geom_mi);
336 void UpdateLineShader(GeometryMaterialInstance& geom_mi);
337 void UpdateUnlitPolygonOffsetShader(GeometryMaterialInstance& geom_mi);
338 utils::EntityInstance<filament::TransformManager>
339 GetGeometryTransformInstance(RenderableGeometry* geom);
340 void CreateSunDirectionalLight();
341 void CreateBackgroundGeometry();
342 void CreateGroundPlaneGeometry();
343
344 std::unordered_map<std::string, RenderableGeometry> geometries_;
345 std::unordered_map<std::string, LightEntity> lights_;
346 std::unordered_map<std::string, std::vector<std::string>> model_geometries_;
347
348 Eigen::Vector4f background_color_;
349 std::shared_ptr<geometry::Image> background_image_;
350 std::string ibl_name_;
351 bool ibl_enabled_ = false;
352 bool skybox_enabled_ = false;
353 std::weak_ptr<filament::IndirectLight> indirect_light_;
354 std::weak_ptr<filament::Skybox> skybox_;
355 IndirectLightHandle ibl_handle_;
356 SkyboxHandle skybox_handle_;
357 LightEntity sun_;
358};
359
360} // namespace rendering
361} // namespace visualization
362} // namespace open3d
std::shared_ptr< core::Tensor > image
Definition: FilamentRenderer.cpp:202
std::function< void(std::shared_ptr< core::Tensor >)> callback
Definition: FilamentRenderer.cpp:201
math::float4 color
Definition: LineSetBuffers.cpp:64
math::float3 position
Definition: LineSetBuffers.cpp:62
A bounding box that is aligned along the coordinate axes.
Definition: BoundingVolume.h:155
The base geometry class for 3D geometries.
Definition: Geometry3D.h:47
The base geometry class.
Definition: Geometry.h:42
A point cloud contains a list of 3D points.
Definition: PointCloud.h:99
Definition: FilamentResourceManager.h:69
static const TextureHandle kDefaultNormalMap
Definition: FilamentResourceManager.h:90
static const TextureHandle kDefaultTexture
Definition: FilamentResourceManager.h:88
void UpdateLightDirection(const std::string &light_name, const Eigen::Vector3f &direction) override
Definition: FilamentScene.cpp:1397
Eigen::Vector3f GetSunLightColor() override
Definition: FilamentScene.cpp:1518
void UpdateLightIntensity(const std::string &light_name, float intensity) override
Definition: FilamentScene.cpp:1409
const std::string & GetIndirectLight() override
Definition: FilamentScene.cpp:1605
void EnableIndirectLight(bool enable) override
Definition: FilamentScene.cpp:1607
~FilamentScene()
Definition: FilamentScene.cpp:177
void SetActiveCamera(const std::string &camera_name) override
Definition: FilamentScene.cpp:307
void SetSunLightColor(const Eigen::Vector3f &color) override
Definition: FilamentScene.cpp:1511
void RenderToDepthImage(std::function< void(std::shared_ptr< geometry::Image >)> callback) override
Size of image is the size of the window.
Definition: FilamentScene.cpp:1838
void SetGeometryTransform(const std::string &object_name, const Transform &transform) override
Definition: FilamentScene.cpp:745
Eigen::Vector3f GetSunLightDirection() override
Definition: FilamentScene.cpp:1554
bool AddPointLight(const std::string &light_name, const Eigen::Vector3f &color, const Eigen::Vector3f &position, float intensity, float falloff, bool cast_shadows) override
Definition: FilamentScene.cpp:1239
void SetSunLightDirection(const Eigen::Vector3f &direction) override
Definition: FilamentScene.cpp:1526
void Draw(filament::Renderer &renderer)
Definition: FilamentScene.cpp:1916
void HideRefractedMaterials(bool hide=true)
Definition: FilamentScene.cpp:1932
void RenderToImage(std::function< void(std::shared_ptr< geometry::Image >)> callback) override
Size of image is the size of the window.
Definition: FilamentScene.cpp:1832
void EnableSunLight(bool enable) override
Definition: FilamentScene.cpp:1479
void SetBackground(const Eigen::Vector4f &color, const std::shared_ptr< geometry::Image > image=nullptr) override
Definition: FilamentScene.cpp:1698
void UpdateLightColor(const std::string &light_name, const Eigen::Vector3f &color) override
Definition: FilamentScene.cpp:1372
Transform GetIndirectLightRotation() override
Definition: FilamentScene.cpp:1640
void EnableLightShadow(const std::string &light_name, bool cast_shadows) override
Definition: FilamentScene.cpp:1437
bool AddDirectionalLight(const std::string &light_name, const Eigen::Vector3f &color, const Eigen::Vector3f &direction, float intensity, bool cast_shadows) override
Definition: FilamentScene.cpp:1317
void SetSunLightIntensity(float intensity) override
Definition: FilamentScene.cpp:1497
geometry::AxisAlignedBoundingBox GetGeometryBoundingBox(const std::string &object_name) override
Definition: FilamentScene.cpp:775
void UpdateLightPosition(const std::string &light_name, const Eigen::Vector3f &position) override
Definition: FilamentScene.cpp:1383
Transform GetGeometryTransform(const std::string &object_name) override
Definition: FilamentScene.cpp:760
void RemoveGeometry(const std::string &object_name) override
Definition: FilamentScene.cpp:690
bool SetIndirectLight(const std::string &ibl_name) override
Definition: FilamentScene.cpp:1562
void SetGroundPlaneColor(const Eigen::Vector4f &color) override
Definition: FilamentScene.cpp:1806
bool GeometryIsVisible(const std::string &object_name) override
Definition: FilamentScene.cpp:719
Light & GetLight(const std::string &light_name) override
Definition: FilamentScene.cpp:1352
void SetIndirectLightIntensity(float intensity) override
Definition: FilamentScene.cpp:1620
ViewHandle AddView(std::int32_t x, std::int32_t y, std::uint32_t w, std::uint32_t h) override
Definition: FilamentScene.cpp:254
void SetSunAngularRadius(float radius) override
Definition: FilamentScene.cpp:1533
float GetSunLightIntensity() override
Definition: FilamentScene.cpp:1504
void UpdateLightFalloff(const std::string &light_name, float falloff) override
Definition: FilamentScene.cpp:1420
void EnableSunLightShadows(bool enable) override
Definition: FilamentScene.cpp:1490
void RemoveView(const ViewHandle &view_id) override
Definition: FilamentScene.cpp:298
bool GetSkyboxVisible() const override
Definition: FilamentScene.cpp:1664
View * GetView(const ViewHandle &view_id) const override
Definition: FilamentScene.cpp:273
void OverrideMaterialAll(const MaterialRecord &material, bool shader_only=true) override
Definition: FilamentScene.cpp:1229
void SetRenderOnce(const ViewHandle &view_id) override
Definition: FilamentScene.cpp:290
Scene * Copy() override
Definition: FilamentScene.cpp:193
void AddCamera(const std::string &camera_name, std::shared_ptr< Camera > cam) override
Definition: FilamentScene.cpp:302
void SetGeometryCulling(const std::string &object_name, bool enable) override
Definition: FilamentScene.cpp:810
void EnableGroundPlane(bool enable, GroundPlane plane) override
Definition: FilamentScene.cpp:1787
void SetSunHaloSize(float size) override
Definition: FilamentScene.cpp:1540
void ShowSkybox(bool show) override
Definition: FilamentScene.cpp:1650
FilamentScene(filament::Engine &engine, FilamentResourceManager &resource_mgr, Renderer &renderer)
Definition: FilamentScene.cpp:166
filament::Scene * GetNativeScene() const
Definition: FilamentScene.h:239
bool AddGeometry(const std::string &object_name, const geometry::Geometry3D &geometry, const MaterialRecord &material, const std::string &downsampled_name="", size_t downsample_threshold=SIZE_MAX) override
Definition: FilamentScene.cpp:324
void UpdateGeometry(const std::string &object_name, const t::geometry::PointCloud &point_cloud, uint32_t update_flags) override
Definition: FilamentScene.cpp:556
void SetGeometryPriority(const std::string &object_name, uint8_t priority) override
Definition: FilamentScene.cpp:822
void SetViewActive(const ViewHandle &view_id, bool is_active) override
Definition: FilamentScene.cpp:282
void ShowGeometry(const std::string &object_name, bool show) override
Definition: FilamentScene.cpp:705
void RemoveLight(const std::string &light_name) override
Definition: FilamentScene.cpp:1363
void SetSunHaloFalloff(float falloff) override
Definition: FilamentScene.cpp:1547
void QueryGeometry(std::vector< std::string > &geometry) override
Definition: FilamentScene.cpp:1223
void SetSunLight(const Eigen::Vector3f &direction, const Eigen::Vector3f &color, float intensity) override
Definition: FilamentScene.cpp:1468
void SetIndirectLightRotation(const Transform &rotation) override
Definition: FilamentScene.cpp:1633
float GetIndirectLightIntensity() override
Definition: FilamentScene.cpp:1626
void GeometryShadows(const std::string &object_name, bool cast_shadows, bool receive_shadows) override
Definition: FilamentScene.cpp:797
Eigen::Transform< float, 3, Eigen::Affine > Transform
Definition: FilamentScene.h:87
void UpdateLightConeAngles(const std::string &light_name, float inner_cone_angle, float outer_cone_angle) override
Definition: FilamentScene.cpp:1431
void RemoveCamera(const std::string &camera_name) override
Definition: FilamentScene.cpp:305
bool HasGeometry(const std::string &object_name) const override
Definition: FilamentScene.cpp:548
void UpdateLight(const std::string &light_name, const Light &light) override
Definition: FilamentScene.cpp:1358
void OverrideMaterial(const std::string &object_name, const MaterialRecord &material) override
Definition: FilamentScene.cpp:1215
bool AddSpotLight(const std::string &light_name, const Eigen::Vector3f &color, const Eigen::Vector3f &position, const Eigen::Vector3f &direction, float intensity, float falloff, float inner_cone_angle, float outer_cone_angle, bool cast_shadows) override
Definition: FilamentScene.cpp:1275
int size
Definition: FilePCD.cpp:59
std::string name
Definition: FilePCD.cpp:58
Definition: FilamentEngine.h:31
const char const char value recording_handle imu_sample recording_handle uint8_t size_t data_size k4a_record_configuration_t config target_format k4a_capture_t capture_handle k4a_imu_sample_t imu_sample playback_handle k4a_logging_message_cb_t void min_level device_handle k4a_imu_sample_t timeout_in_ms capture_handle capture_handle capture_handle image_handle temperature_c k4a_image_t image_handle uint8_t image_handle image_handle image_handle image_handle uint32_t
Definition: K4aPlugin.cpp:567
const char const char value recording_handle imu_sample recording_handle uint8_t size_t data_size k4a_record_configuration_t config target_format k4a_capture_t capture_handle k4a_imu_sample_t imu_sample playback_handle k4a_logging_message_cb_t void min_level device_handle k4a_imu_sample_t int32_t
Definition: K4aPlugin.cpp:414
REHandle< EntityType::IndexBuffer > IndexBufferHandle
Definition: RendererHandle.h:158
REHandle< EntityType::VertexBuffer > VertexBufferHandle
Definition: RendererHandle.h:157
REHandle< EntityType::MaterialInstance > MaterialInstanceHandle
Definition: RendererHandle.h:154
REHandle< EntityType::Skybox > SkyboxHandle
Definition: RendererHandle.h:151
REHandle< EntityType::IndirectLight > IndirectLightHandle
Definition: RendererHandle.h:150
Definition: PinholeCameraIntrinsic.cpp:35