oh man, that’s really great! The AA pass keeps all of the detail of scalefx-hybrid but smooths all of the stippled edges. Beautiful!
1 Like
Meanwhile i tried (for some time, if i’m honest) to do a good deblur setup. Until recently i lacked a proper aa shader and deblur had it’s issues, but i managed to resolve them now.
I’m posting the shader/preset if anyone cares to try it out. 
Here is a comparison with the above preset agains deblured:
http://screenshotcomparison.com/comparison/119330
deblur.glsl (goes in deblur/shaders/)
/*
Deblur Shader
Copyright (C) 2005 - 2018 guest(r) - [email protected]
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#pragma parameter OFFSET "Deblur offset" 2.0 1.0 4.0 0.25
#pragma parameter DEBLUR "Deblur str. " 2.5 1.0 7.0 0.25
#pragma parameter SMART "Smart deblur " 0.0 0.0 1.0 1.00
#if defined(VERTEX)
#if __VERSION__ >= 130
#define COMPAT_VARYING out
#define COMPAT_ATTRIBUTE in
#define COMPAT_TEXTURE texture
#else
#define COMPAT_VARYING varying
#define COMPAT_ATTRIBUTE attribute
#define COMPAT_TEXTURE texture2D
#endif
#ifdef GL_ES
#define COMPAT_PRECISION mediump
#else
#define COMPAT_PRECISION
#endif
COMPAT_ATTRIBUTE vec4 VertexCoord;
COMPAT_ATTRIBUTE vec4 COLOR;
COMPAT_ATTRIBUTE vec4 TexCoord;
COMPAT_VARYING vec4 COL0;
COMPAT_VARYING vec4 TEX0;
vec4 _oPosition1;
uniform mat4 MVPMatrix;
uniform COMPAT_PRECISION int FrameDirection;
uniform COMPAT_PRECISION int FrameCount;
uniform COMPAT_PRECISION vec2 OutputSize;
uniform COMPAT_PRECISION vec2 TextureSize;
uniform COMPAT_PRECISION vec2 InputSize;
// compatibility #defines
#define vTexCoord TEX0.xy
#define SourceSize vec4(TextureSize, 1.0 / TextureSize) //either TextureSize or InputSize
#define OutSize vec4(OutputSize, 1.0 / OutputSize)
void main()
{
gl_Position = MVPMatrix * VertexCoord;
TEX0.xy = TexCoord.xy * 1.00001;
}
#elif defined(FRAGMENT)
#ifdef GL_ES
#ifdef GL_FRAGMENT_PRECISION_HIGH
precision highp float;
#else
precision mediump float;
#endif
#define COMPAT_PRECISION mediump
#else
#define COMPAT_PRECISION
#endif
#if __VERSION__ >= 130
#define COMPAT_VARYING in
#define COMPAT_TEXTURE texture
out COMPAT_PRECISION vec4 FragColor;
#else
#define COMPAT_VARYING varying
#define FragColor gl_FragColor
#define COMPAT_TEXTURE texture2D
#endif
uniform COMPAT_PRECISION int FrameDirection;
uniform COMPAT_PRECISION int FrameCount;
uniform COMPAT_PRECISION vec2 OutputSize;
uniform COMPAT_PRECISION vec2 TextureSize;
uniform COMPAT_PRECISION vec2 InputSize;
uniform sampler2D Texture;
COMPAT_VARYING vec4 TEX0;
// compatibility #defines
#define Source Texture
#define vTexCoord TEX0.xy
#define SourceSize vec4(TextureSize, 1.0 / TextureSize) //either TextureSize or InputSize
#define OutSize vec4(OutputSize, 1.0 / OutputSize)
#ifdef PARAMETER_UNIFORM
uniform COMPAT_PRECISION float OFFSET;
uniform COMPAT_PRECISION float DEBLUR;
uniform COMPAT_PRECISION float SMART;
#else
#define OFFSET 2.0
#define DEBLUR 2.5
#define SMART 0.0
#endif
vec3 dt = vec3(1.,1.,1.);
vec3 dtt = vec3(0.0001, 0.0001, 0.0001);
void main()
{
vec2 tex = vTexCoord;
vec2 texsize = SourceSize.xy;
float dx = OFFSET/texsize.x;
float dy = OFFSET/texsize.y;
vec4 yx = vec4( dx, dy,-dx,-dy);
vec2 xx = vec2( dx, 0.0);
vec2 yy = vec2( 0.0, dy);
vec3 c11 = COMPAT_TEXTURE(Source, tex ).xyz;
vec3 c00 = COMPAT_TEXTURE(Source, tex + yx.zw).xyz;
vec3 c20 = COMPAT_TEXTURE(Source, tex + yx.xw).xyz;
vec3 c22 = COMPAT_TEXTURE(Source, tex + yx.xy).xyz;
vec3 c02 = COMPAT_TEXTURE(Source, tex + yx.zy).xyz;
vec3 c10 = COMPAT_TEXTURE(Source, tex - yy ).xyz;
vec3 c21 = COMPAT_TEXTURE(Source, tex + xx ).xyz;
vec3 c12 = COMPAT_TEXTURE(Source, tex + yy ).xyz;
vec3 c01 = COMPAT_TEXTURE(Source, tex - xx ).xyz;
vec3 mn1 = min (min (c00,c01),c02);
vec3 mn2 = min (min (c10,c11),c12);
vec3 mn3 = min (min (c20,c21),c22);
vec3 mx1 = max (max (c00,c01),c02);
vec3 mx2 = max (max (c10,c11),c12);
vec3 mx3 = max (max (c20,c21),c22);
vec3 d11;
mn1 = min(min(mn1,mn2),mn3);
mx1 = max(max(mx1,mx2),mx3);
vec3 dif1 = abs(c11-mn1) + dtt;
vec3 dif2 = abs(c11-mx1) + dtt;
float DB1 = DEBLUR; float dif;
if (SMART == 1.0)
{
dif = max(length(dif1),length(dif2));
dif = min(dif, 1.0);
DB1 = max(mix(-0.5, DEBLUR, dif), 1.0);
}
dif1=vec3(pow(dif1.x,DB1),pow(dif1.y,DB1),pow(dif1.z,DB1));
dif2=vec3(pow(dif2.x,DB1),pow(dif2.y,DB1),pow(dif2.z,DB1));
d11 = vec3((dif1.x*mx1.x + dif2.x*mn1.x)/(dif1.x + dif2.x),
(dif1.y*mx1.y + dif2.y*mn1.y)/(dif1.y + dif2.y),
(dif1.z*mx1.z + dif2.z*mn1.z)/(dif1.z + dif2.z));
float k10 = 1.0/(dot(abs(c10-d11),dt)+0.0001);
float k01 = 1.0/(dot(abs(c01-d11),dt)+0.0001);
float k11 = 1.0/(dot(abs(c11-d11),dt)+0.0001);
float k21 = 1.0/(dot(abs(c21-d11),dt)+0.0001);
float k12 = 1.0/(dot(abs(c12-d11),dt)+0.0001);
c11 = (k10*c10 + k01*c01 + k11*c11 + k21*c21 + k12*c12)/(k10+k01+k11+k21+k12);
FragColor = vec4(c11,1.0);
}
#endif
Has some options and can even do smart deblur, like the 4xsoftSdB.
The preset is like above (the hybrid really doesen’t produce bad shapes): 
xsoft+scalefx-hybrid+level2aa-sharp.glslp
shaders = 9
shader0 = ../scalefx/shaders/scalefx-pass0.glsl
filter_linear0 = false
scale_type0 = source
scale0 = 1.0
float_framebuffer0 = true
shader1 = ../scalefx/shaders/scalefx-pass1.glsl
filter_linear1 = false
scale_type1 = source
scale1 = 1.0
float_framebuffer1 = true
shader2 = ../scalefx/shaders/scalefx-pass2.glsl
filter_linear2 = false
scale_type2 = source
scale2 = 1.0
shader3 = ../scalefx/shaders/scalefx-pass3.glsl
filter_linear3 = false
scale_type3 = source
scale3 = 1.0
shader4 = ../scalefx/shaders/scalefx-pass4-hybrid.glsl
filter_linear4 = false
scale_type4 = source
scale4 = 3.0
shader5 = ../stock.glsl
filter_linear5 = false
scale_type5 = source
scale5 = 2.0
shader6 = ../anti-aliasing/shaders/aa-shader-4.0-level2/aa-shader-4.0-level2-pass1-noblend.glsl
filter_linear6 = true
scale_type6 = source
scale6 = 1.0
shader7 = ../anti-aliasing/shaders/aa-shader-4.0-level2/aa-shader-4.0-level2-pass2.glsl
filter_linear7 = true
scale_type7 = source
scale7 = 1.0
shader8 = ../deblur/shaders/deblur.glsl
filter_linear8 = true
scale_type8 = source
scale8 = 1.0
parameters = "SFX_CLR;SFX_SAA"
SFX_SAA = 0.00
SFX_CLR = 0.60
Now i’m off to grind some rep. 
Edit: Much better smart version (option) of deblur.
1 Like
nice work! It reminds me of when I take my glasses off/on 
1 Like
This reminds me of self-similarity / fractal image upscaling. You may find some of the papers on it inspiring.
good stuff guest.r! I’m actually working on this currently. thing is that pixelart interpolation and reverseAA don’t really mix since both require the unscaled source image. the hybrid version of ScaleFX is really just a dirty hack and not a proper merge of both algorithms. but I’m confident to say that I’ve come up with a solution that will replace the hybrid approach. this is a post process shader that you can simply add on top of ScaleFX (or other filters for that matter).
ScaleFX
ScaleFX-Hybrid
ScaleFX + rAA_post
this is with no additional shaders. just ScaleFX and a generalized rAA version I’m working on. I’ll post more soon.
2 Likes
so after thinking it through I came to the conclusion that reverseAA can’t really be properly combined with edge interpolation filters. while there is a hybrid version of ScaleFX it is a little rough. so instead of continuing this hybrid approach I decided to make a post-process version of rAA which can be applied to an already upscaled image. therefore it can be used with other filters as well not only ScaleFX but xBR or others too.
this rAA version works on a 1x scale factor and assumes the image is already a 3x upscale. just add the two passes on top of your shader stack (that results in a 3x image) and you’re good to go. the filter is less performant than the original rAA since it requires more pixel fetches. beside the known sharpness parameter there is a new “gradient protection” one that well protects pixels that don’t need changing.
right now I have only a cg version. if someone wants to help out translating it to the other shader languages, be my guest. I included a new shader preset in the scalefx folder which demonstrates it and makes the hybrid version obsolete. just extract the archive to your retroarch folder and you’re done.
ScaleFX
ScaleFX + rAA
1 Like
That looks awesome. I’ll see about converting to glsl/slang ASAP.
@MetinSeven You can now use RetroArch to process images via CLI. Set the shader you want to use, then exit RetroArch and use the --max-frames-ss switch (you can set it for something pretty low, like 10). You can use a simple bash script like this to process all of the images in a directory:
#!/bin/bash
for i in *.jpg;
do
retroarch --max-frames-ss 10 --max-frames-ss-path ~/pics "$i"
done
1 Like
Thanks @hunterk! Being kind of a Terminal illiterate I hope I can get it running. I’ll give it a try.
Ok, just FYI then: I think to run RetroArch from CLI on OSX, you’ll need to navigate into the app bundle and use the actual executable therein.
1 Like
Looks great!! Congrats!
Any more examples?
I uploaded the Cg version to the common-shaders repo and a GLSL version to its repo. I made a slang port but something’s up with it. With d3d11 driver, it doesn’t appear to do anything (no impact on framerate), while with Vulkan it seems to crash the whole driver 
The high quality edge interpolation and detected AA thinning / blending got me thinking, could this handle rotating images?
The most popular pixel art rotating program to my knowledge is Rotsprite, and if I remember correctly it works by enlarging the input to 8x or 16x using “something similar to Scale2x”, rotates it nearest neighbor, and then scales it back down nearest neighbor.
It seems like this has some knowledge of aesthetically viable or important slopes which I imagine would be valuable in reconstructing an image at an angle and could perhaps snap slopes to closely matching yet better looking slopes.
I can’t imagine rotating the image being useful for a screen shader though, the only non 90 degree rotated game I can think of is Rhyme Rider Kerorican on the WonderSwan Color.
I love this shader but I currently see a problem when I try to use it with the DeSmuME core and the option to utilize the Hybrid view which shows one of the DS screens large with both screens smaller on the side. I don’t know how feasible it would be but could you modify or create a separate shader that would modify one portion of the screen more than the rest so that the main screen has a higher scale factor?
Here is an example of the original image:
And here it is with ScaleNx:
The two small displays on the side do look like they have improved with ScaleNx, but because the large display is at a different scale it rounds the pixels instead of being a scaled up version of the small display.
I already started the discussion in github. A callback is needed to output a partial framebuffer.
For reasons alluded to by papermanzero it doesn’t work the way you’d hope it does. Shaders don’t see the two DS screens separately, and upscaling shaders get confused by hybrid mode because one of the screens has already been nearest-neighbour upscaled by the emu core.
As a workaround, you must go into core options of DeSmuME and set the internal resolution to 256x192 (1x) and set hybrid layout scale to 1. Unfortunately the small screens will be distorted.
I just pushed up slang/glsl presets for a shader that takes desmume’s regular output, applies scalefx and then arranges the screens into a hybrid-view layout. They’re in handheld > ds-hybrid-scalefx.
*the ds-hybrid-view shader has been around for a while, but it had some weird behavior that I fixed and it plays nicely with more shaders now.
1 Like
http://imaging.cs.msu.ru/en/research/warping This technology appears to be similar to your new rAA, you might get new ideas out of it. Gist is that it sharpens by pulling pixels together at edges, rather than modifying the pixel values themselves.
2 Likes
There’s a fringe of what reminds me of Photoshop’s color burn where some pixels are adjusted by the reverse AA shader. At first I thought maybe it was intended and looked better with some things but now I’m more of the opinion that it’s an error as it’s putting really dark, saturated, and “burnt” borders on some shapes rather than just shrinking areas of intermediate colors like in most situations.
The effect is especially visible when applied to a nearest neighbor 3x upscale. In the ScaleFx 3x upscale it’s especially bad on the wrist guard, under the hair on the forehead, stomach, pants waist, top of the shoulders, under the eyes, the nose, and somehow an errant strip of burnt pixels on the cheek at the left eye just floating and not connected to any obvious shape.
Whatever is causing this may be best as a toggle since the Killer Instinct screenshots didn’t have any obvious problems when I last saw them, and I looked at them a lot while tabbing between them.