ScaleNx - Artifact Removal and Algorithm Improvement

[guest.r]

Meanwhile i tried (for some time, if i’m honest) to do a good deblur setup. Until recently i lacked a proper aa shader and deblur had it’s issues, but i managed to resolve them now.

I’m posting the shader/preset if anyone cares to try it out.

Here is a comparison with the above preset agains deblured:

http://screenshotcomparison.com/comparison/119330

deblur.glsl (goes in deblur/shaders/)

/*
   Deblur Shader
   
   Copyright (C) 2005 - 2018 guest(r) - [email protected]

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public License
   as published by the Free Software Foundation; either version 2
   of the License, or (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
*/

#pragma parameter OFFSET "Deblur offset" 2.0 1.0 4.0 0.25 
#pragma parameter DEBLUR "Deblur str.  " 2.5 1.0 7.0 0.25 
#pragma parameter SMART  "Smart deblur " 0.0 0.0 1.0 1.00 

#if defined(VERTEX)

#if __VERSION__ >= 130
#define COMPAT_VARYING out
#define COMPAT_ATTRIBUTE in
#define COMPAT_TEXTURE texture
#else
#define COMPAT_VARYING varying 
#define COMPAT_ATTRIBUTE attribute 
#define COMPAT_TEXTURE texture2D
#endif

#ifdef GL_ES
#define COMPAT_PRECISION mediump
#else
#define COMPAT_PRECISION
#endif

COMPAT_ATTRIBUTE vec4 VertexCoord;
COMPAT_ATTRIBUTE vec4 COLOR;
COMPAT_ATTRIBUTE vec4 TexCoord;
COMPAT_VARYING vec4 COL0;
COMPAT_VARYING vec4 TEX0;

vec4 _oPosition1; 
uniform mat4 MVPMatrix;
uniform COMPAT_PRECISION int FrameDirection;
uniform COMPAT_PRECISION int FrameCount;
uniform COMPAT_PRECISION vec2 OutputSize;
uniform COMPAT_PRECISION vec2 TextureSize;
uniform COMPAT_PRECISION vec2 InputSize;

// compatibility #defines
#define vTexCoord TEX0.xy
#define SourceSize vec4(TextureSize, 1.0 / TextureSize) //either TextureSize or InputSize
#define OutSize vec4(OutputSize, 1.0 / OutputSize)

void main()
{
    gl_Position = MVPMatrix * VertexCoord;
    TEX0.xy = TexCoord.xy * 1.00001;
}

#elif defined(FRAGMENT)

#ifdef GL_ES
#ifdef GL_FRAGMENT_PRECISION_HIGH
precision highp float;
#else
precision mediump float;
#endif
#define COMPAT_PRECISION mediump
#else
#define COMPAT_PRECISION
#endif

#if __VERSION__ >= 130
#define COMPAT_VARYING in
#define COMPAT_TEXTURE texture
out COMPAT_PRECISION vec4 FragColor;
#else
#define COMPAT_VARYING varying
#define FragColor gl_FragColor
#define COMPAT_TEXTURE texture2D
#endif

uniform COMPAT_PRECISION int FrameDirection;
uniform COMPAT_PRECISION int FrameCount;
uniform COMPAT_PRECISION vec2 OutputSize;
uniform COMPAT_PRECISION vec2 TextureSize;
uniform COMPAT_PRECISION vec2 InputSize;
uniform sampler2D Texture;
COMPAT_VARYING vec4 TEX0;

// compatibility #defines
#define Source Texture
#define vTexCoord TEX0.xy

#define SourceSize vec4(TextureSize, 1.0 / TextureSize) //either TextureSize or InputSize
#define OutSize vec4(OutputSize, 1.0 / OutputSize)

#ifdef PARAMETER_UNIFORM
uniform COMPAT_PRECISION float OFFSET;
uniform COMPAT_PRECISION float DEBLUR;
uniform COMPAT_PRECISION float SMART;
#else
#define OFFSET  2.0
#define DEBLUR  2.5
#define SMART   0.0
#endif 

vec3  dt = vec3(1.,1.,1.);
vec3  dtt = vec3(0.0001, 0.0001, 0.0001); 

void main()
{
   vec2 tex = vTexCoord;	
   vec2 texsize = SourceSize.xy;
   float dx = OFFSET/texsize.x;
   float dy = OFFSET/texsize.y;
   
   vec4 yx = vec4( dx, dy,-dx,-dy);
   vec2 xx = vec2( dx, 0.0);
   vec2 yy = vec2( 0.0, dy);
   
   vec3 c11 = COMPAT_TEXTURE(Source, tex        ).xyz;  
   vec3 c00 = COMPAT_TEXTURE(Source, tex + yx.zw).xyz;
   vec3 c20 = COMPAT_TEXTURE(Source, tex + yx.xw).xyz;
   vec3 c22 = COMPAT_TEXTURE(Source, tex + yx.xy).xyz;
   vec3 c02 = COMPAT_TEXTURE(Source, tex + yx.zy).xyz;
   vec3 c10 = COMPAT_TEXTURE(Source, tex - yy   ).xyz;     
   vec3 c21 = COMPAT_TEXTURE(Source, tex + xx   ).xyz;
   vec3 c12 = COMPAT_TEXTURE(Source, tex + yy   ).xyz;
   vec3 c01 = COMPAT_TEXTURE(Source, tex - xx   ).xyz;
   
   vec3 mn1 = min (min (c00,c01),c02);
   vec3 mn2 = min (min (c10,c11),c12);
   vec3 mn3 = min (min (c20,c21),c22);
   vec3 mx1 = max (max (c00,c01),c02);
   vec3 mx2 = max (max (c10,c11),c12);
   vec3 mx3 = max (max (c20,c21),c22);

   vec3 d11;
   
   mn1 = min(min(mn1,mn2),mn3);
   mx1 = max(max(mx1,mx2),mx3);

   vec3 dif1 = abs(c11-mn1) + dtt;
   vec3 dif2 = abs(c11-mx1) + dtt;
   
   float DB1 = DEBLUR; float dif;
   
   if (SMART == 1.0)
   {
       dif = max(length(dif1),length(dif2));
	   dif = min(dif, 1.0);
       DB1 = max(mix(-0.5, DEBLUR, dif), 1.0);
   }
   
   dif1=vec3(pow(dif1.x,DB1),pow(dif1.y,DB1),pow(dif1.z,DB1));
   dif2=vec3(pow(dif2.x,DB1),pow(dif2.y,DB1),pow(dif2.z,DB1));

   d11 = vec3((dif1.x*mx1.x + dif2.x*mn1.x)/(dif1.x + dif2.x),
                (dif1.y*mx1.y + dif2.y*mn1.y)/(dif1.y + dif2.y),
                (dif1.z*mx1.z + dif2.z*mn1.z)/(dif1.z + dif2.z));   
 
   float k10 = 1.0/(dot(abs(c10-d11),dt)+0.0001);  
   float k01 = 1.0/(dot(abs(c01-d11),dt)+0.0001);
   float k11 = 1.0/(dot(abs(c11-d11),dt)+0.0001);  
   float k21 = 1.0/(dot(abs(c21-d11),dt)+0.0001);
   float k12 = 1.0/(dot(abs(c12-d11),dt)+0.0001);   
   
   c11 = (k10*c10 + k01*c01 + k11*c11 + k21*c21 + k12*c12)/(k10+k01+k11+k21+k12);
   
   FragColor = vec4(c11,1.0); 
} 
#endif

Has some options and can even do smart deblur, like the 4xsoftSdB.

The preset is like above (the hybrid really doesen’t produce bad shapes):

xsoft+scalefx-hybrid+level2aa-sharp.glslp

shaders = 9

shader0 = ../scalefx/shaders/scalefx-pass0.glsl
filter_linear0 = false
scale_type0 = source
scale0 = 1.0
float_framebuffer0 = true

shader1 = ../scalefx/shaders/scalefx-pass1.glsl
filter_linear1 = false
scale_type1 = source
scale1 = 1.0
float_framebuffer1 = true

shader2 = ../scalefx/shaders/scalefx-pass2.glsl
filter_linear2 = false
scale_type2 = source
scale2 = 1.0

shader3 = ../scalefx/shaders/scalefx-pass3.glsl
filter_linear3 = false
scale_type3 = source
scale3 = 1.0

shader4 = ../scalefx/shaders/scalefx-pass4-hybrid.glsl
filter_linear4 = false
scale_type4 = source
scale4 = 3.0 

shader5 = ../stock.glsl
filter_linear5 = false
scale_type5 = source
scale5 = 2.0

shader6 = ../anti-aliasing/shaders/aa-shader-4.0-level2/aa-shader-4.0-level2-pass1-noblend.glsl
filter_linear6 = true
scale_type6 = source
scale6 = 1.0

shader7 = ../anti-aliasing/shaders/aa-shader-4.0-level2/aa-shader-4.0-level2-pass2.glsl
filter_linear7 = true
scale_type7 = source
scale7 = 1.0

shader8 = ../deblur/shaders/deblur.glsl
filter_linear8 = true
scale_type8 = source
scale8 = 1.0

parameters = "SFX_CLR;SFX_SAA"
SFX_SAA = 0.00
SFX_CLR = 0.60

Now i’m off to grind some rep.

Edit: Much better smart version (option) of deblur.

[hunterk]

nice work! It reminds me of when I take my glasses off/on

[torridgristle]

This reminds me of self-similarity / fractal image upscaling. You may find some of the papers on it inspiring.

[Sp00kyFox]

good stuff guest.r! I’m actually working on this currently. thing is that pixelart interpolation and reverseAA don’t really mix since both require the unscaled source image. the hybrid version of ScaleFX is really just a dirty hack and not a proper merge of both algorithms. but I’m confident to say that I’ve come up with a solution that will replace the hybrid approach. this is a post process shader that you can simply add on top of ScaleFX (or other filters for that matter).

ScaleFX

ScaleFX-Hybrid

ScaleFX + rAA_post

this is with no additional shaders. just ScaleFX and a generalized rAA version I’m working on. I’ll post more soon.

[Sp00kyFox]

so after thinking it through I came to the conclusion that reverseAA can’t really be properly combined with edge interpolation filters. while there is a hybrid version of ScaleFX it is a little rough. so instead of continuing this hybrid approach I decided to make a post-process version of rAA which can be applied to an already upscaled image. therefore it can be used with other filters as well not only ScaleFX but xBR or others too.

this rAA version works on a 1x scale factor and assumes the image is already a 3x upscale. just add the two passes on top of your shader stack (that results in a 3x image) and you’re good to go. the filter is less performant than the original rAA since it requires more pixel fetches. beside the known sharpness parameter there is a new “gradient protection” one that well protects pixels that don’t need changing.

right now I have only a cg version. if someone wants to help out translating it to the other shader languages, be my guest. I included a new shader preset in the scalefx folder which demonstrates it and makes the hybrid version obsolete. just extract the archive to your retroarch folder and you’re done.

download

ScaleFX

ScaleFX + rAA

[hunterk]

That looks awesome. I’ll see about converting to glsl/slang ASAP.

@MetinSeven You can now use RetroArch to process images via CLI. Set the shader you want to use, then exit RetroArch and use the --max-frames-ss switch (you can set it for something pretty low, like 10). You can use a simple bash script like this to process all of the images in a directory:

#!/bin/bash
for i in *.jpg;
do
retroarch --max-frames-ss 10 --max-frames-ss-path ~/pics "$i"
done

[MetinSeven]

Thanks @hunterk! Being kind of a Terminal illiterate I hope I can get it running. I’ll give it a try.

[hunterk]

Ok, just FYI then: I think to run RetroArch from CLI on OSX, you’ll need to navigate into the app bundle and use the actual executable therein.

[MetinSeven]

Thanks again @hunterk! I’ll try it a.s.a.p…

[Hyllian]

Looks great!! Congrats!

Any more examples?

[Sp00kyFox]

Original 3x

ScaleFX

ScaleFX + rAA

[hunterk]

I uploaded the Cg version to the common-shaders repo and a GLSL version to its repo. I made a slang port but something’s up with it. With d3d11 driver, it doesn’t appear to do anything (no impact on framerate), while with Vulkan it seems to crash the whole driver

[torridgristle]

The high quality edge interpolation and detected AA thinning / blending got me thinking, could this handle rotating images?

The most popular pixel art rotating program to my knowledge is Rotsprite, and if I remember correctly it works by enlarging the input to 8x or 16x using “something similar to Scale2x”, rotates it nearest neighbor, and then scales it back down nearest neighbor.

It seems like this has some knowledge of aesthetically viable or important slopes which I imagine would be valuable in reconstructing an image at an angle and could perhaps snap slopes to closely matching yet better looking slopes.

I can’t imagine rotating the image being useful for a screen shader though, the only non 90 degree rotated game I can think of is Rhyme Rider Kerorican on the WonderSwan Color.

[tsudico]

I love this shader but I currently see a problem when I try to use it with the DeSmuME core and the option to utilize the Hybrid view which shows one of the DS screens large with both screens smaller on the side. I don’t know how feasible it would be but could you modify or create a separate shader that would modify one portion of the screen more than the rest so that the main screen has a higher scale factor?

Here is an example of the original image:

And here it is with ScaleNx:

The two small displays on the side do look like they have improved with ScaleNx, but because the large display is at a different scale it rounds the pixels instead of being a scaled up version of the small display.

[papermanzero]

I already started the discussion in github. A callback is needed to output a partial framebuffer.

Issue: Native Resolution in Hybrid Mode results in distorted small screens

[milesrotaru1]

For reasons alluded to by papermanzero it doesn’t work the way you’d hope it does. Shaders don’t see the two DS screens separately, and upscaling shaders get confused by hybrid mode because one of the screens has already been nearest-neighbour upscaled by the emu core.

As a workaround, you must go into core options of DeSmuME and set the internal resolution to 256x192 (1x) and set hybrid layout scale to 1. Unfortunately the small screens will be distorted.

[hunterk]

I just pushed up slang/glsl presets for a shader that takes desmume’s regular output, applies scalefx and then arranges the screens into a hybrid-view layout. They’re in handheld > ds-hybrid-scalefx.

*the ds-hybrid-view shader has been around for a while, but it had some weird behavior that I fixed and it plays nicely with more shaders now.

[torridgristle]

http://imaging.cs.msu.ru/en/research/warping This technology appears to be similar to your new rAA, you might get new ideas out of it. Gist is that it sharpens by pulling pixels together at edges, rather than modifying the pixel values themselves.

[torridgristle]

There’s a fringe of what reminds me of Photoshop’s color burn where some pixels are adjusted by the reverse AA shader. At first I thought maybe it was intended and looked better with some things but now I’m more of the opinion that it’s an error as it’s putting really dark, saturated, and “burnt” borders on some shapes rather than just shrinking areas of intermediate colors like in most situations.

[Album] Imgur

The effect is especially visible when applied to a nearest neighbor 3x upscale. In the ScaleFx 3x upscale it’s especially bad on the wrist guard, under the hair on the forehead, stomach, pants waist, top of the shoulders, under the eyes, the nose, and somehow an errant strip of burnt pixels on the cheek at the left eye just floating and not connected to any obvious shape.

Whatever is causing this may be best as a toggle since the Killer Instinct screenshots didn’t have any obvious problems when I last saw them, and I looked at them a lot while tabbing between them.

[Sp00kyFox]

yeah this is an issue with reverse AA itself. it causes these outliers sometimes when dealing with content that doesn’t have smooth gradients but vibrant contrasting colors. have to analyze the algorithm to see what the issue is. right now I just wouldn’t recommend it systems like GBC or NES. and these games usually don’t really benefit from it anyways. but I’m currently working on an update for this filter so I’m definitely looking into this as well. thanks for your post!

edit: I think it is caused by doing interpolation vectors in RGB space which sometimes causes weird gradients that don’t match with human perception. might be better to do it all channel-wise instead or use another color space. have to experiment there.

edit2: I was mistaken. rAA already works channel-wise.