Filter visible sources by overlap with screen interval using screenScales.get(0)

src/main/java/bdv/viewer/render/MultiResolutionRenderer.java

@@ -28,7 +28,8 @@
  */
 package bdv.viewer.render;
 
-import bdv.viewer.RequestRepaint;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.ExecutorService;
 
 import net.imglib2.Interval;
@@ -41,6 +42,8 @@ import net.imglib2.util.Intervals;
 
 import bdv.cache.CacheControl;
 import bdv.util.MovingAverage;
+import bdv.viewer.RequestRepaint;
+import bdv.viewer.SourceAndConverter;
 import bdv.viewer.ViewerState;
 import bdv.viewer.render.ScreenScales.IntervalRenderData;
 import bdv.viewer.render.ScreenScales.ScreenScale;
@@ -152,9 +155,14 @@ public class MultiResolutionRenderer
 	private ViewerState currentViewerState;
 
 	/**
-	 * How many sources are visible in {@link #currentViewerState}.
+	 * The sources that are actually visible on screen currently. This means
+	 * that the sources both are visible in the {@link #currentViewerState} (via
+	 * {@link ViewerState#getVisibleAndPresentSources()
+	 * getVisibleAndPresentSources}) and, when transformed to viewer
+	 * coordinates, overlap the screen area ({@link #display}).
 	 */
-	private int currentNumVisibleSources;
+	private final List< SourceAndConverter< ? > > currentVisibleSourcesOnScreen;
+
 
 	/**
 	 * The last successfully rendered (not cancelled) full frame result.
@@ -278,6 +286,7 @@ public class MultiResolutionRenderer
 		this.painterThread = painterThread;
 		projector = null;
 		currentScreenScaleIndex = -1;
+		currentVisibleSourcesOnScreen = new ArrayList<>();
 		screenScales = new ScreenScales( screenScaleFactors, targetRenderNanos );
 		renderStorage = new RenderStorage();
 
@@ -351,6 +360,7 @@ public class MultiResolutionRenderer
 		projector = null;
 		currentViewerState = null;
 		currentRenderResult = null;
+		currentVisibleSourcesOnScreen.clear();
 		renderStorage.clear();
 	}
 
@@ -384,7 +394,8 @@ public class MultiResolutionRenderer
 			if ( newInterval )
 			{
 				intervalMode = true;
-				final double renderNanosPerPixel = renderNanosPerPixelAndSource.getAverage() * currentNumVisibleSources;
+				final int numSources = currentVisibleSourcesOnScreen.size();
+				final double renderNanosPerPixel = renderNanosPerPixelAndSource.getAverage() * numSources;
 				requestedIntervalScaleIndex = screenScales.suggestIntervalScreenScale( renderNanosPerPixel, currentScreenScaleIndex );
 			}
 
@@ -410,8 +421,9 @@ public class MultiResolutionRenderer
 		if ( newFrame )
 		{
 			currentViewerState = viewerState.snapshot();
-			currentNumVisibleSources = currentViewerState.getVisibleAndPresentSources().size();
-			final double renderNanosPerPixel = renderNanosPerPixelAndSource.getAverage() * currentNumVisibleSources;
+			VisibilityUtils.computeVisibleSourcesOnScreen( currentViewerState, screenScales.get( 0 ), currentVisibleSourcesOnScreen );
+			final int numSources = currentVisibleSourcesOnScreen.size();
+			final double renderNanosPerPixel = renderNanosPerPixelAndSource.getAverage() * numSources;
 			requestedScreenScaleIndex = screenScales.suggestScreenScale( renderNanosPerPixel );
 		}
 
@@ -445,8 +457,8 @@ public class MultiResolutionRenderer
 				renderResult.setScaleFactor( screenScale.scale() );
 				currentViewerState.getViewerTransform( renderResult.getViewerTransform() );
 
-				renderStorage.checkRenewData( screenScales.get( 0 ).width(), screenScales.get( 0 ).height(), currentNumVisibleSources );
-				projector = createProjector( currentViewerState, requestedScreenScaleIndex, renderResult.getTargetImage(), 0, 0 );
+				renderStorage.checkRenewData( screenScales.get( 0 ).width(), screenScales.get( 0 ).height(), currentVisibleSourcesOnScreen.size() );
+				projector = createProjector( currentViewerState, currentVisibleSourcesOnScreen, requestedScreenScaleIndex, renderResult.getTargetImage(), 0, 0 );
 				requestNewFrameIfIncomplete = projectorFactory.requestNewFrameIfIncomplete();
 			}
 			p = projector;
@@ -496,7 +508,7 @@ public class MultiResolutionRenderer
 			{
 				intervalResult.init( intervalRenderData.width(), intervalRenderData.height() );
 				intervalResult.setScaleFactor( intervalRenderData.scale() );
-				projector = createProjector( currentViewerState, requestedIntervalScaleIndex, intervalResult.getTargetImage(), intervalRenderData.offsetX(), intervalRenderData.offsetY() );
+				projector = createProjector( currentViewerState, currentVisibleSourcesOnScreen, requestedIntervalScaleIndex, intervalResult.getTargetImage(), intervalRenderData.offsetX(), intervalRenderData.offsetY() );
 			}
 			p = projector;
 		}
@@ -543,7 +555,8 @@ public class MultiResolutionRenderer
 
 	private void recordRenderTime( final RenderResult result, final long renderNanos )
 	{
-		final int numRenderPixels = ( int ) Intervals.numElements( result.getTargetImage() ) * currentNumVisibleSources;
+		final int numSources = currentVisibleSourcesOnScreen.size();
+		final int numRenderPixels = ( int ) Intervals.numElements( result.getTargetImage() ) * numSources;
 		if ( numRenderPixels >= 4096 )
 			renderNanosPerPixelAndSource.add( renderNanos / ( double ) numRenderPixels );
 	}
@@ -597,6 +610,7 @@ public class MultiResolutionRenderer
 
 	private VolatileProjector createProjector(
 			final ViewerState viewerState,
+			final List< SourceAndConverter< ? > > visibleSourcesOnScreen,
 			final int screenScaleIndex,
 			final RandomAccessibleInterval< ARGBType > screenImage,
 			final int offsetX,
@@ -609,6 +623,7 @@ public class MultiResolutionRenderer
 
 		final VolatileProjector projector = projectorFactory.createProjector(
 				viewerState,
+				visibleSourcesOnScreen,
 				screenImage,
 				screenTransform,
 				renderStorage );

src/main/java/bdv/viewer/render/ProjectorFactory.java

@@ -110,13 +110,15 @@ class ProjectorFactory
 
 	/**
 	 * Create a projector for rendering the specified {@code ViewerState} to the
-	 * specified {@code screenImage}, with the current visible sources and
+	 * specified {@code screenImage}, with the current visible sources (visible
+	 * in {@code ViewerState} and actually currently visible on screen) and
 	 * timepoint of the {@code ViewerState}, and the specified
 	 * {@code screenTransform} from global coordinates to coordinates in the
 	 * {@code screenImage}.
 	 */
 	public VolatileProjector createProjector(
 			final ViewerState viewerState,
+			final List< SourceAndConverter< ? > > visibleSourcesOnScreen,
 			final RandomAccessibleInterval< ARGBType > screenImage,
 			final AffineTransform3D screenTransform,
 			final RenderStorage renderStorage )
@@ -131,22 +133,20 @@ class ProjectorFactory
 		final int width = ( int ) screenImage.dimension( 0 );
 		final int height = ( int ) screenImage.dimension( 1 );
 
-		final ArrayList< SourceAndConverter< ? > > visibleSources = new ArrayList<>( viewerState.getVisibleAndPresentSources() );
-		visibleSources.sort( viewerState.sourceOrder() );
 		VolatileProjector projector;
-		if ( visibleSources.isEmpty() )
+		if ( visibleSourcesOnScreen.isEmpty() )
 			projector = new EmptyProjector<>( screenImage );
-		else if ( visibleSources.size() == 1 )
+		else if ( visibleSourcesOnScreen.size() == 1 )
 		{
 			final byte[] maskArray = renderStorage.getMaskArray( 0 );
-			projector = createSingleSourceProjector( viewerState, visibleSources.get( 0 ), screenImage, screenTransform, maskArray );
+			projector = createSingleSourceProjector( viewerState, visibleSourcesOnScreen.get( 0 ), screenImage, screenTransform, maskArray );
 		}
 		else
 		{
 			final ArrayList< VolatileProjector > sourceProjectors = new ArrayList<>();
 			final ArrayList< RandomAccessibleInterval< ARGBType > > sourceImages = new ArrayList<>();
 			int j = 0;
-			for ( final SourceAndConverter< ? > source : visibleSources )
+			for ( final SourceAndConverter< ? > source : visibleSourcesOnScreen )
 			{
 				final RandomAccessibleInterval< ARGBType > renderImage = renderStorage.getRenderImage( width, height, j );
 				final byte[] maskArray = renderStorage.getMaskArray( j );
@@ -155,7 +155,7 @@ class ProjectorFactory
 				sourceProjectors.add( p );
 				sourceImages.add( renderImage );
 			}
-			projector = accumulateProjectorFactory.createProjector( sourceProjectors, visibleSources, sourceImages, screenImage, numRenderingThreads, renderingExecutorService );
+			projector = accumulateProjectorFactory.createProjector( sourceProjectors, visibleSourcesOnScreen, sourceImages, screenImage, numRenderingThreads, renderingExecutorService );
 		}
 		previousTimepoint = viewerState.getCurrentTimepoint();
 		return projector;

src/main/java/bdv/viewer/render/VisibilityUtils.java

+package bdv.viewer.render;
+
+import bdv.util.MipmapTransforms;
+import bdv.viewer.Interpolation;
+import bdv.viewer.Source;
+import bdv.viewer.SourceAndConverter;
+import bdv.viewer.ViewerState;
+import java.util.List;
+import java.util.Set;
+import net.imglib2.FinalRealInterval;
+import net.imglib2.Interval;
+import net.imglib2.realtransform.AffineTransform3D;
+
+class VisibilityUtils
+{
+	/**
+	 * Compute a list of sources are currently visible on screen.
+	 * <p>
+	 * This means that the sources
+	 * <ul>
+	 *     <li>are visible in the given {@code ViewerState}, and,</li>
+	 *     <li>when transformed to viewer coordinates, overlap the screen area.</li>
+	 * </ul>
+	 * The returned list of sources is sorted by {@code viewerState.sourceOrder()}.
+	 *
+	 * @param viewerState
+	 * 		specifies sources, transform, and current timepoint
+	 * @param screenScale
+	 * 		specifies screen size and scale transform
+	 * @param result
+	 * 		list of currently visible sources is stored here
+	 */
+	static void computeVisibleSourcesOnScreen(
+			final ViewerState viewerState,
+			final ScreenScales.ScreenScale screenScale,
+			final List< SourceAndConverter< ? > > result )
+	{
+		result.clear();
+
+		final int screenMinX = 0;
+		final int screenMinY = 0;
+		final int screenMaxX = screenScale.width() - 1;
+		final int screenMaxY = screenScale.height() - 1;
+
+		final AffineTransform3D screenTransform = viewerState.getViewerTransform();
+		screenTransform.preConcatenate( screenScale.scaleTransform() );
+
+		final AffineTransform3D sourceToScreen = new AffineTransform3D();
+		final double[] sourceMin = new double[ 3 ];
+		final double[] sourceMax = new double[ 3 ];
+
+		final Set< SourceAndConverter< ? > > sources = viewerState.getVisibleAndPresentSources();
+		final int t = viewerState.getCurrentTimepoint();
+		final double expand = viewerState.getInterpolation() == Interpolation.NEARESTNEIGHBOR ? 0.5 : 1.0;
+
+		for ( final SourceAndConverter< ? > source : sources )
+		{
+			final Source< ? > spimSource = source.getSpimSource();
+			final int level = MipmapTransforms.getBestMipMapLevel( screenTransform, spimSource, t );
+			spimSource.getSourceTransform( t, level, sourceToScreen );
+			sourceToScreen.preConcatenate( screenTransform );
+
+			final Interval interval = spimSource.getSource( t, level );
+			for ( int d = 0; d < 3; d++ )
+			{
+				sourceMin[ d ] = interval.realMin( d ) - expand;
+				sourceMax[ d ] = interval.realMax( d ) + expand;
+			}
+			final FinalRealInterval bb = sourceToScreen.estimateBounds( new FinalRealInterval( sourceMin, sourceMax ) );
+
+			if ( bb.realMax( 0 ) >= screenMinX
+					&& bb.realMin( 0 ) <= screenMaxX
+					&& bb.realMax( 1 ) >= screenMinY
+					&& bb.realMin( 1 ) <= screenMaxY
+					&& bb.realMax( 2 ) >= 0
+					&& bb.realMin( 2 ) <= 0 )
+			{
+				result.add( source );
+			}
+		}
+
+		result.sort( viewerState.sourceOrder() );
+	}
+	// TODO: Eventually, for thousands of sources, this could be moved to ViewerState,
+	//  in order to avoid creating a new intermediate HashSet for every
+	//  viewerState.getVisibleAndPresentSources().
+	//  However, other issues will become bottlenecks before that, e.g.,
+	//  copying source list when taking snapshots of ViewerState every frame,
+	//  painting MultiBoxOverlay, etc.
+}