diff --git a/examples/ipywidgets/ipywidgets_modify_image.py b/examples/ipywidgets/ipywidgets_modify_image.py index c0206e945..7d8d7deef 100644 --- a/examples/ipywidgets/ipywidgets_modify_image.py +++ b/examples/ipywidgets/ipywidgets_modify_image.py @@ -1,5 +1,5 @@ """ -ipwidgets modify an ImageGraphic +ipywidgets modify an ImageGraphic ================================ Use ipywidgets to modify some features of an ImageGraphic. Run in jupyterlab. diff --git a/examples/scenes/multiobject_tracking.py b/examples/scenes/multiobject_tracking.py new file mode 100644 index 000000000..d027197a2 --- /dev/null +++ b/examples/scenes/multiobject_tracking.py @@ -0,0 +1,106 @@ +""" +Vizualizing Multi-Object Tracking +================================= + +By sharing the scene between subplots, we can dynamically add detail views for each detected object. +""" +# %% imports +import fastplotlib as fpl +import numpy as np +import imageio.v3 as iio + + + +# %% Mock data +# TODO: replace with YOLO-tracking/detection example? + + +movie = iio.imread("imageio:cockatoo.mp4") +# make shape (t, h, w, 1) (for monochrome examples) +movie = movie[:,:,:, 0] # take one channel +print(movie.shape) + +# boxes are are x,y,w,h... but there can be multiple per frame - so it's a list of boxes that can also be empty. +center_path = np.array([[movie.shape[2]/2 + 200*np.sin(t/30), movie.shape[1]/2 + 50*np.cos(t/5)] for t in range(len(movie)-60)]) +width = 80 + 20*np.sin(np.linspace(0, 10*np.pi, len(movie)-60)) +height = 100 + 30*np.cos(np.linspace(0, 8*np.pi, len(movie)-60)) +bboxes = {t+30: [[center_path[t][0]-width[t]/2, center_path[t][1]-height[t]/2, width[t], height[t]]] for t in range(len(movie)-60)} +# add a bit of jitter noise and some dropouts +fake_preds = {} +for t in bboxes: + fake_preds[t] = [box.copy() for box in bboxes[t]] + for box in fake_preds[t]: + box[0] += np.random.normal(scale=5.0) + box[1] += np.random.normal(scale=5.0) + box[2] += np.random.normal(scale=3.0) + box[3] += np.random.normal(scale=8.0) + # fake_preds[t] = [box.copy() for box in bboxes[t]] + if np.random.rand() < 0.1: + fake_preds[t] = [] + +def get_lines(box) -> fpl.LineGraphic: + # convert boxes to line graphics so they can be added to a LineGraphicCollection during runtime. + lines = [] + x, y, w, h = box + lines.append([x-0.5*w, y+0.5*h]) # top left + lines.append([x+0.5*w, y+0.5*h]) # top right + lines.append([x+0.5*w, y-0.5*h]) # bottom right + lines.append([x-0.5*w, y-0.5*h]) # bottom left + lines.append(lines[0]) # close box + return fpl.LineGraphic(np.array(lines), alpha=0.95) + + +# %% make the plot +global iw +# TODO: can this constructor be done with just one data? (multiple views?) +iw = fpl.ImageWidget(data=[movie,movie], rgb=False, cmap="white", figure_shape=(1,2), figure_kwargs={"size" : (1400, 650), "controller_ids": [[0,1]], "scene_ids": "sync"}, graphic_kwargs={"vmin": 0, "vmax": 255}) +# TODO: remove the 2nd Histogram tool (it's shared now.) + +iw._image_widget_sliders._fps["t"] = 30 # not exposed +figure: fpl.Figure = iw.figure +# can these be zero init too? +box_overlay = figure[0,0].add_line_collection(data=[]) + +# TODO: in the right subplot, add one detail box per detected object--- automatically open and close them? + +figure[0,1].axes.visible = False +figure[0,1].toolbar = False + +def update_overlay(index): + t = index["t"] + + # clean up old boxes: + for g in box_overlay.graphics: + box_overlay.remove_graphic(g) + # pass + + label_boxes = bboxes.get(t, []) + for label in label_boxes: + lines = get_lines(label) + lines.colors = "red" # red for labels + box_overlay.add_graphic(lines) + + pred_boxes = fake_preds.get(t, []) + for pred in pred_boxes: + lines = get_lines(pred) + lines.colors = "green" # green for predictions + # maybe get confidence for alpha or something fun + box_overlay.add_graphic(lines) + + # update crop: + if label_boxes: + long_edge = max(label_boxes[0][2], label_boxes[0][3]) # here we lose multiple objects + # TODO: make this behave better (based on a relative value?) + scale = 100 / long_edge + figure[0,1].camera.set_state({"position": (label_boxes[0][0], label_boxes[0][1], -1), "zoom": scale}) + + +iw.add_event_handler(update_overlay) +iw.show() + + +# NOTE: fpl.loop.run() should not be used for interactive sessions +# See the "JupyterLab and IPython" section in the user guide +if __name__ == "__main__": + print(__doc__) + fpl.loop.run() diff --git a/examples/scenes/scene_by_int.py b/examples/scenes/scene_by_int.py new file mode 100644 index 000000000..ce67d7b50 --- /dev/null +++ b/examples/scenes/scene_by_int.py @@ -0,0 +1,56 @@ +""" +Specify Scene IDs with integers +========================= + +Share scenes between subplots using integer IDs +""" +# derived from specify_integers.py + +# test_example = false +# sphinx_gallery_pygfx_docs = 'screenshot' + +import numpy as np +import fastplotlib as fpl + + +xs = np.linspace(0, 2 * np.pi, 100) +sine = np.sin(xs) +cosine = np.cos(xs) + +# scene IDs +# one scene is created for each unique ID +# if the IDs are the same, those subplots will present the same scene +ids = [ + [0, 1], + [2, 0], +] + +figure = fpl.Figure( + shape=(2, 2), + scene_ids=ids, + size=(700, 560), +) + +for subplot, scene_id in zip(figure, np.asarray(ids).ravel()): + subplot.title = f"scene id: {scene_id}" + +figure[0, 0].add_line(np.column_stack([xs, sine])) + +figure[0, 1].add_line(np.random.rand(100)) +figure[1, 0].add_line(np.random.rand(100)) + +# since we the scene from [0,0], we don't need add anything here +# in fact adding something here would add it to both. +# figure[1, 1].add_line(np.column_stack([xs, cosine])) + +figure[1, 1].axes.visible = False # hide axes as this is a tad silly. + + +figure.show(maintain_aspect=False) + + +# NOTE: fpl.loop.run() should not be used for interactive sessions +# See the "JupyterLab and IPython" section in the user guide +if __name__ == "__main__": + print(__doc__) + fpl.loop.run() diff --git a/fastplotlib/layouts/_figure.py b/fastplotlib/layouts/_figure.py index 8fd5dc666..1b3c81c03 100644 --- a/fastplotlib/layouts/_figure.py +++ b/fastplotlib/layouts/_figure.py @@ -47,6 +47,11 @@ def __init__( | Iterable[Iterable[str]] ) = None, controllers: pygfx.Controller | Iterable[Iterable[pygfx.Controller]] = None, + scene_ids: ( + Iterable[int] + | Iterable[Iterable[int]] + | Iterable[Iterable[str]] + ) = None, canvas: str | BaseRenderCanvas | pygfx.Texture = None, renderer: pygfx.WgpuRenderer = None, canvas_kwargs: dict = None, @@ -103,6 +108,10 @@ def __init__( | list of lists of subplot names, each sublist is synced: [[subplot_a, subplot_b, subplot_e], [subplot_c, subplot_d]] | this syncs subplot_a, subplot_b and subplot_e together; syncs subplot_c and subplot_d together + scene_ids: str, list of int, np.ndarray of int, or list with sublists of subplot str names, optional + | If `None` a unique scene is created for each subplot + | If "sync" all the subplots use the same scene + controllers: pygfx.Controller | list[pygfx.Controller] | np.ndarray[pygfx.Controller], optional Directly provide pygfx.Controller instances(s). Useful if you want to use a ``Controller`` from an existing subplot or a ``Controller`` you have already instantiated. Also useful if you want to provide a custom @@ -392,6 +401,86 @@ def __init__( for cam in cams[1:]: _controller.add_camera(cam) + # lot's of code borrowed from controller_ids above... + # parse scene_ids to make the required scenes and match them to subplots + if scene_ids is None: + subplot_scenes = np.array([None] * n_subplots) + # if None, we can just let the scene creation be hadnled in the Subplot/Plot Area further down. + else: + if isinstance(scene_ids, str): + if scene_ids == "sync": + # this will end up creating one scene to be used by every subplot + scene_ids = np.zeros(n_subplots, dtype=int) + else: + raise ValueError( + f"`scene_ids` must be one of 'sync', an array/list of subplot names, or an array/list of " + f"integer ids. You have passed: {scene_ids}.\n" + f"See the docstring for more details." + ) + + # list scene_ids + elif isinstance(scene_ids, (list, np.ndarray)): + ids_flat = list(chain(*scene_ids)) + + # list of str of subplot names, convert this to integer ids + if all([isinstance(item, str) for item in ids_flat]): + if subplot_names is None: + raise ValueError( + "must specify subplot `names` to use list of str for `scene_ids`" + ) + + # make sure each controller_id str is a subplot name + if not all([n in subplot_names for n in ids_flat]): + raise KeyError( + f"all `scene_ids` strings must be one of the subplot names. You have passed " + f"the following `scene_ids`:\n{scene_ids}\n\n" + f"and the following subplot names:\n{subplot_names}" + ) + + if len(ids_flat) > len(set(ids_flat)): + raise ValueError( + f"id strings must not appear twice in `scene_ids`: \n{scene_ids}" + ) + + # initialize scene_ids array + ids_init = np.arange(n_subplots) + + # set id based on subplot position for each synced sublist + for row_ix, sublist in enumerate(scene_ids): + for name in sublist: + ids_init[subplot_names == name] = -( + row_ix + 1 + ) # use negative numbers to avoid collision with positive numbers from np.arange + + scene_ids = ids_init + + # integer ids + elif all([isinstance(item, (int, np.integer)) for item in ids_flat]): + scene_ids = np.asarray(scene_ids).flatten() + if scene_ids.max() < 0: + raise ValueError( + f"if passing an integer array of `scene_ids`, " + f"all the integers must be positive:{scene_ids}" + ) + + else: + raise TypeError( + f"list argument to `scene_ids` must be a list of `str` or `int`, " + f"you have passed: {scene_ids}" + ) + + if scene_ids.size != n_subplots: + raise ValueError( + f"Number of scene_ids: {scene_ids.size} " + f"does not match the number of subplots: {n_subplots}" + ) + + # make the real scenes for each subplot + subplot_scenes = np.empty(shape=n_subplots, dtype=object) + for sid in np.unique(scene_ids): + _scene = pygfx.Scene() + subplot_scenes[scene_ids == sid] = _scene + self._canvas = canvas self._renderer = renderer @@ -410,6 +499,7 @@ def __init__( for i in range(n_subplots): camera = subplot_cameras[i] controller = subplot_controllers[i] + scene = subplot_scenes[i] if subplot_names is not None: name = subplot_names[i] @@ -421,6 +511,7 @@ def __init__( camera=camera, controller=controller, canvas=canvas, + scene=scene, renderer=renderer, name=name, rect=rects[i], diff --git a/fastplotlib/layouts/_imgui_figure.py b/fastplotlib/layouts/_imgui_figure.py index 046c622ea..242829022 100644 --- a/fastplotlib/layouts/_imgui_figure.py +++ b/fastplotlib/layouts/_imgui_figure.py @@ -39,6 +39,11 @@ def __init__( | Iterable[Iterable[str]] ) = None, controllers: pygfx.Controller | Iterable[Iterable[pygfx.Controller]] = None, + scene_ids: ( + Iterable[int] + | Iterable[Iterable[int]] + | Iterable[Iterable[str]] + ) = None, canvas: str | BaseRenderCanvas | pygfx.Texture = None, renderer: pygfx.WgpuRenderer = None, canvas_kwargs: dict = None, @@ -56,6 +61,7 @@ def __init__( controller_types=controller_types, controller_ids=controller_ids, controllers=controllers, + scene_ids=scene_ids, canvas=canvas, renderer=renderer, canvas_kwargs=canvas_kwargs, diff --git a/fastplotlib/layouts/_subplot.py b/fastplotlib/layouts/_subplot.py index 73f669fe5..ea991106a 100644 --- a/fastplotlib/layouts/_subplot.py +++ b/fastplotlib/layouts/_subplot.py @@ -19,6 +19,7 @@ def __init__( camera: Literal["2d", "3d"] | pygfx.PerspectiveCamera, controller: pygfx.Controller | str, canvas: BaseRenderCanvas | pygfx.Texture, + scene: pygfx.Scene = None, rect: np.ndarray = None, extent: np.ndarray = None, resizeable: bool = True, @@ -48,6 +49,9 @@ def __init__( canvas: BaseRenderCanvas, or a pygfx.Texture Provides surface on which a scene will be rendered. + scene: pygfx.Scene, optional + scene this subplot will render, if ``None``, a new scene will be created + renderer: WgpuRenderer object used to render scenes using wgpu @@ -71,7 +75,7 @@ def __init__( parent=parent, camera=camera, controller=controller, - scene=pygfx.Scene(), + scene=scene if scene else pygfx.Scene(), canvas=canvas, renderer=renderer, name=name, @@ -83,8 +87,12 @@ def __init__( self.docks[pos] = dv self.children.append(dv) - self._axes = Axes(self) - self.scene.add(self.axes.world_object) + if not any([isinstance(c, Axes) for c in self.scene.children]): + # Create axes only if no scene is provided + # this doesn't find axes deeper in the scene graph... but just assume they are top level for now. + # TODO: rethink how axes are handled in the scene per subplot... + self._axes = Axes(self) + self.scene.add(self.axes.world_object) self._frame = Frame( viewport=self.viewport,