Browse Source

added keyboard parser

master
sciion 4 years ago
parent
commit
a6deda1c8a
No account linked to committer's email address

+ 28
- 0
README.md View File

@@ -0,0 +1,28 @@
1
+# GUI-Agent
2
+"Scripting user activity is a hard thing to do" - me
3
+
4
+## About
5
+This project aims at creating a system for scripting user activity inside computer environments using computer vision and peripheral devices. The technologies used are the following
6
+
7
+- OpenCV
8
+- Jython
9
+- Java
10
+- GraphLib
11
+
12
+Tons of inspiration was taken from the Sikuli project and until I get on good terms with VirtualBox SDK this project can be seen as a inferior version of Sikuli.
13
+
14
+## Install
15
+All you have to do are the following esoteric tasks taken right out of M. C. Escher’s wildest nightmares:
16
+- Install java
17
+- Download OpenCV
18
+- Make sure the OpenCV library is present in your library path
19
+- Git clone this repo
20
+- Done
21
+
22
+## Use
23
+
24
+The primary way this project is meant to be used involves creating jython scripts that interacts with the function calls in the API.java file. Some example scripts are present in the project and can be executed using the following command
25
+```
26
+java -jar lib\jython.jar scripts\<filename>.py
27
+```
28
+More features are expected to be added as the project proceeds. The primary focus right now is to make the software somewhat useable.

BIN
assets/templates/template-firefox.png View File


BIN
assets/templates/template-sublime.png View File


BIN
lib/AgentAPI.jar View File


+ 47
- 61
project/src/me/sciion/agent/api/API.java View File

@@ -1,114 +1,100 @@
1 1
 package me.sciion.agent.api;
2 2
 
3
-import java.awt.AWTException;
4
-import java.awt.Robot;
5
-import java.awt.event.InputEvent;
6
-import java.awt.event.MouseEvent;
7 3
 import java.awt.image.BufferedImage;
8 4
 import java.awt.image.DataBufferByte;
9
-import java.io.File;
10
-import java.io.IOException;
11 5
 import java.util.Arrays;
12 6
 
13
-import javax.imageio.ImageIO;
14
-import javax.swing.text.Document;
15
-
16 7
 import org.opencv.core.Core;
17 8
 import org.opencv.core.CvType;
18 9
 import org.opencv.imgproc.Imgproc;
19
-import org.w3c.dom.Node;
20 10
 
21 11
 import me.sciion.agent.Run;
12
+import me.sciion.agent.environments.HostSystem;
13
+import me.sciion.agent.environments.VirtualEnvironment;
22 14
 import me.sciion.agent.matching.Match;
23 15
 import me.sciion.agent.matching.Matcher;
24 16
 import me.sciion.agent.state.Template;
25
-import me.sciion.agent.utils.KeySequence;
26 17
 import me.sciion.agent.utils.Location;
27 18
 
28 19
 public class API {
29
-  
20
+
30 21
     static {
31 22
 	System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
23
+	setEnvironment(new HostSystem());
32 24
     }
33 25
     public static final String VERSION = "Version: 0.1";
34 26
     private static Matcher matcher = new Matcher(Imgproc.TM_CCOEFF_NORMED);
35
-    
36
-    public static Template loadTemplate(String path){
37
-	
27
+
28
+    private static VirtualEnvironment environment;
29
+
30
+    public static void setEnvironment(VirtualEnvironment env) {
31
+	environment = env;
32
+    }
33
+
34
+    public static Template loadTemplate(String path) {
35
+
38 36
 	BufferedImage templateImage = Run.getImageResource(path);
39
-	return new Template("Undefined template id",Imgproc.TM_CCOEFF_NORMED, 0.97, path,templateImage);
37
+	return new Template("Undefined template id", Imgproc.TM_CCOEFF_NORMED, 0.97, path, templateImage);
40 38
     }
41
-    
42
-    public static boolean exists(Template template){
43
-	Match m = matcher.match(Run.cvtMat(Run.getScreenshot(BufferedImage.TYPE_3BYTE_BGR),CvType.CV_8UC3), template);
39
+
40
+    public static boolean exists(Template template) {
41
+	Match m = matcher.match(Run.cvtMat(environment.getScreenshot(), CvType.CV_8UC3), template);
44 42
 	return m.getScore() >= template.getThreshold();
45 43
     }
46
-    
47
-    public static void waitForChange(){
48
-	 int hash = 0;
49
-	 int hash2 = -1;
50
-	   BufferedImage first = Run.getScreenshot(BufferedImage.TYPE_3BYTE_BGR);
51
-	do{
44
+
45
+    public static void waitForChange() {
46
+	int hash = 0;
47
+	int hash2 = -1;
48
+	BufferedImage first = environment.getScreenshot();
49
+	do {
52 50
 	    try {
53 51
 		Thread.sleep(1);
54 52
 	    } catch (InterruptedException e) {
55 53
 		e.printStackTrace();
56 54
 	    }
57
-	    BufferedImage second = Run.getScreenshot(BufferedImage.TYPE_3BYTE_BGR);
55
+	    BufferedImage second = environment.getScreenshot();
58 56
 	    hash = Arrays.hashCode(((DataBufferByte) first.getRaster().getDataBuffer()).getData());
59 57
 	    hash2 = Arrays.hashCode(((DataBufferByte) second.getRaster().getDataBuffer()).getData());
60
-	}while(hash == hash2);
58
+	} while (hash == hash2);
61 59
 	return;
62 60
 
63 61
     }
64
-    
65
-    public static boolean waitForTemplate(Template template){
62
+
63
+    public static boolean waitForTemplate(Template template) {
66 64
 	Match m;
67
-	do{
68
-	    m = matcher.match(Run.cvtMat(Run.getScreenshot(BufferedImage.TYPE_3BYTE_BGR),CvType.CV_8UC3), template);
65
+	do {
66
+	    m = matcher.match(Run.cvtMat(environment.getScreenshot(), CvType.CV_8UC3), template);
69 67
 	    try {
70 68
 		Thread.sleep(1);
71 69
 	    } catch (InterruptedException e) {
72 70
 		e.printStackTrace();
73 71
 	    }
74
-	}while(m.getScore() < template.getThreshold());
72
+	} while (m.getScore() < template.getThreshold());
75 73
 	return true;
76 74
     }
77
-    
78
-    public static Location locate(Template template){
79
-	Match m = matcher.match(Run.cvtMat(Run.getScreenshot(BufferedImage.TYPE_3BYTE_BGR),CvType.CV_8UC3), template);
80
-	return new Location(m.getX()+template.getImage().getWidth()/2,m.getY()+template.getImage().getHeight()/2);
75
+
76
+    public static Location locate(Template template) {
77
+	Match m = matcher.match(Run.cvtMat(environment.getScreenshot(), CvType.CV_8UC3), template);
78
+	return new Location(m.getX() + template.getImage().getWidth() / 2,
79
+		m.getY() + template.getImage().getHeight() / 2);
81 80
     }
82
-    
83
-    public static void click(){
84
-	try {
85
-	    
86
-	    new Robot().mousePress(InputEvent.BUTTON1_DOWN_MASK);
87
-	    Thread.sleep(1);
88
-	    new Robot().mouseRelease(InputEvent.BUTTON1_MASK); 
89
-	} catch (AWTException e) {
90
-	    e.printStackTrace();
91
-	} catch (InterruptedException e) {
92
-	    e.printStackTrace();
93
-	}
81
+
82
+    public static void click() {
83
+	environment.rightClick();
94 84
     }
95
-    
96
-    public static void move(Location location){
97
-	try {
98
-	    new Robot().mouseMove(location.x, location.y);
99
-	} catch (AWTException e) {
100
-	    e.printStackTrace();
101
-	}
85
+
86
+    public static void move(Location location) {
87
+	environment.move(location);
102 88
     }
103
-    
104
-    public static void type(KeySequence sequence){
105
-	
89
+
90
+    public static void type(String line) {
91
+	environment.type(line);
106 92
     }
107
-    
93
+
108 94
     public static String getVersion() {
109 95
 	return VERSION;
110 96
     }
111
-    
112
-    public static void main(String[] args){
97
+    public static void main(String [] args){
98
+	
113 99
     }
114 100
 }

+ 42
- 8
project/src/me/sciion/agent/environments/HostSystem.java View File

@@ -1,12 +1,20 @@
1 1
 package me.sciion.agent.environments;
2 2
 
3 3
 import java.awt.AWTException;
4
+import java.awt.Rectangle;
4 5
 import java.awt.Robot;
6
+import java.awt.Toolkit;
5 7
 import java.awt.event.InputEvent;
6
-import java.awt.event.MouseEvent;
8
+import java.awt.event.KeyEvent;
9
+import java.awt.image.BufferedImage;
10
+import java.io.File;
11
+import java.io.IOException;
7 12
 
8
-import me.sciion.agent.utils.KeySequence;
13
+import javax.imageio.ImageIO;
14
+
15
+import me.sciion.agent.utils.Key;
9 16
 import me.sciion.agent.utils.Location;
17
+import me.sciion.agent.utils.parser.KeyParser;
10 18
 
11 19
 public class HostSystem implements VirtualEnvironment{
12 20
 
@@ -15,6 +23,7 @@ public class HostSystem implements VirtualEnvironment{
15 23
     public HostSystem() {
16 24
 	try {
17 25
 	    robot = new Robot();
26
+	    robot.setAutoDelay(50);
18 27
 	} catch (AWTException e) {
19 28
 	    e.printStackTrace();
20 29
 	}
@@ -33,14 +42,39 @@ public class HostSystem implements VirtualEnvironment{
33 42
     }
34 43
 
35 44
     @Override
36
-    public void type(KeySequence sequence) {
37
-	
38
-    }
45
+    public void type(String line) {
46
+	KeyParser parser = new KeyParser();
47
+	for(Key k: parser.parse(line)){
48
+	    if(k.isPress())
49
+		robot.keyPress(k.getCode());
50
+	    else
51
+		robot.keyRelease(k.getCode());
52
+	}
53
+	robot.keyPress(KeyEvent.VK_ENTER);
54
+	robot.keyRelease(KeyEvent.VK_ENTER);
55
+
56
+}
39 57
 
40 58
     @Override
41 59
     public void move(Location location) {
42
-	// TODO Auto-generated method stub
43
-	
60
+	robot.mouseMove(location.x, location.y);
61
+    }
62
+    
63
+    @Override
64
+    public BufferedImage getScreenshot() {
65
+	Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
66
+	BufferedImage raw = new BufferedImage(screenRect.width, screenRect.height, BufferedImage.TYPE_3BYTE_BGR);
67
+	try {
68
+		BufferedImage tmp = new Robot().createScreenCapture(screenRect);
69
+		File f = File.createTempFile("tmpImage","png");
70
+		ImageIO.write(tmp, "png", f);
71
+		tmp = ImageIO.read(f);
72
+		raw.getGraphics().drawImage(tmp, 0, 0, null);
73
+	} catch (AWTException e) {
74
+		e.printStackTrace();
75
+	} catch (IOException e) {
76
+	    e.printStackTrace();
77
+	}
78
+       return raw;
44 79
     }
45
-
46 80
 }

+ 4
- 3
project/src/me/sciion/agent/environments/VirtualEnvironment.java View File

@@ -1,6 +1,7 @@
1 1
 package me.sciion.agent.environments;
2 2
 
3
-import me.sciion.agent.utils.KeySequence;
3
+import java.awt.image.BufferedImage;
4
+
4 5
 import me.sciion.agent.utils.Location;
5 6
 
6 7
 public interface VirtualEnvironment {
@@ -8,7 +9,7 @@ public interface VirtualEnvironment {
8 9
     
9 10
    public void rightClick();
10 11
    public void leftClick();
11
-   public void type(KeySequence sequence);
12
+   public void type(String line);
12 13
    public void move(Location location);
13
-   
14
+   public BufferedImage getScreenshot();
14 15
 }

+ 0
- 12
project/src/me/sciion/agent/perifials/Keyboard.java View File

@@ -1,12 +0,0 @@
1
-package me.sciion.agent.perifials;
2
-
3
-import me.sciion.agent.environments.VirtualEnvironment;
4
-
5
-public class Keyboard implements VirtualDevice{
6
-
7
-    @Override
8
-    public void interact(VirtualEnvironment virtualEnvironment) {
9
-	virtualEnvironment.
10
-    }
11
-
12
-}

+ 0
- 8
project/src/me/sciion/agent/perifials/VirtualDevice.java View File

@@ -1,8 +0,0 @@
1
-package me.sciion.agent.perifials;
2
-
3
-import me.sciion.agent.environments.VirtualEnvironment;
4
-
5
-public interface VirtualDevice {
6
-
7
-    public void interact(VirtualEnvironment virtualEnvironment);
8
-}

+ 0
- 5
project/src/me/sciion/agent/utils/CharToKeyEvent.java View File

@@ -1,5 +0,0 @@
1
-package me.sciion.agent.utils;
2
-
3
-public class CharToKeyEvent {
4
-
5
-}

+ 23
- 0
project/src/me/sciion/agent/utils/Key.java View File

@@ -0,0 +1,23 @@
1
+package me.sciion.agent.utils;
2
+
3
+public class Key {
4
+
5
+    private boolean press;
6
+    private int code;
7
+    
8
+    public Key(int code, boolean press){
9
+	this.code = code;
10
+	this.press = press;
11
+    }
12
+
13
+    public boolean isPress() {
14
+        return press;
15
+    }
16
+
17
+    public int getCode() {
18
+        return code;
19
+    }
20
+    
21
+    
22
+    
23
+}

+ 0
- 16
project/src/me/sciion/agent/utils/KeySequence.java View File

@@ -1,16 +0,0 @@
1
-package me.sciion.agent.utils;
2
-
3
-public class KeySequence {
4
-
5
-    
6
-    private String sequence;
7
-    
8
-    public KeySequence(String keys){
9
-	sequence = keys;
10
-    }
11
-    
12
-    public String getSequence(){
13
-	return  sequence;
14
-    }
15
-    
16
-}

+ 109
- 0
project/src/me/sciion/agent/utils/parser/KeyParser.java View File

@@ -0,0 +1,109 @@
1
+package me.sciion.agent.utils.parser;
2
+
3
+import java.awt.AWTException;
4
+import java.awt.Robot;
5
+import java.awt.event.KeyEvent;
6
+import java.util.LinkedList;
7
+import java.util.Vector;
8
+
9
+import me.sciion.agent.utils.Key;
10
+
11
+public class KeyParser {
12
+    
13
+    private LinkedList<Token> tokens;
14
+    private Vector<Key> keyCodes;
15
+    public KeyParser() {
16
+	tokens = new LinkedList<Token>();
17
+	keyCodes = new Vector<Key>();
18
+    }
19
+
20
+    public Vector<Key> parse(String line) {
21
+	tokens.clear();
22
+	KeyScanner scanner = new KeyScanner(line);
23
+
24
+	Token t;
25
+	do {
26
+	    t = scanner.next();
27
+	    if (t == null)
28
+		break;
29
+	    tokens.addLast(t);
30
+	} while (t != null);
31
+	eval();
32
+	return keyCodes;
33
+    }
34
+
35
+    public void eval() {
36
+	if (tokens.isEmpty())
37
+	    return;
38
+	Token token;
39
+	do {
40
+	    token = tokens.poll();
41
+	    try {
42
+		evalModifier(token);
43
+	    } catch (Exception e) {
44
+		e.printStackTrace();
45
+	    }
46
+	} while (token != null);
47
+    }
48
+
49
+    private void evalLiteral(Token t) throws Exception {
50
+	if (t.getType() == TokenType.LITERAL) {
51
+	    int c = (int) Character.toUpperCase(t.getValue());
52
+	    if (Character.isUpperCase(t.getValue())) {
53
+		keyCodes.add(new Key(KeyEvent.VK_SHIFT, true));
54
+		keyCodes.add(new Key(c, true));
55
+		keyCodes.add(new Key(c, false));
56
+		keyCodes.add(new Key(KeyEvent.VK_SHIFT, false));
57
+	    } else {
58
+		keyCodes.add(new Key(c, true));
59
+		keyCodes.add(new Key(c, false));
60
+	    }
61
+
62
+	} else {
63
+	    throw new Exception();
64
+	}
65
+    }
66
+
67
+    private void evalModifier(Token t) throws Exception {
68
+	if (t == null)
69
+	    return;
70
+	if (t.getType() == TokenType.MODIFIER) {
71
+	    if (t.getValue() == 'M') {
72
+		keyCodes.add(new Key(KeyEvent.VK_ALT, true));
73
+		evalModifier(tokens.poll());
74
+		keyCodes.add(new Key(KeyEvent.VK_ALT, false));
75
+	    } else if (t.getValue() == 'C') {
76
+		keyCodes.add(new Key(KeyEvent.VK_CONTROL, true));
77
+		evalModifier(tokens.poll());
78
+		keyCodes.add(new Key(KeyEvent.VK_CONTROL, false));
79
+	    }else if (t.getValue() == 'W') {
80
+		keyCodes.add(new Key(KeyEvent.VK_WINDOWS, true));
81
+		evalModifier(tokens.poll());
82
+		keyCodes.add(new Key(KeyEvent.VK_WINDOWS, false));
83
+	    }else if (t.getValue() == 'S') {
84
+		keyCodes.add(new Key(KeyEvent.VK_SHIFT, true));
85
+		evalModifier(tokens.poll());
86
+		keyCodes.add(new Key(KeyEvent.VK_SHIFT, false));
87
+	    }
88
+	} else if (t.getType() == TokenType.LITERAL) {
89
+	    evalLiteral(t);
90
+	} else
91
+	    throw new Exception();
92
+    }
93
+    public static void main(String [] args){
94
+	KeyParser parser = new KeyParser();
95
+	Robot r = null ;
96
+	try {
97
+	    r = new Robot();
98
+	} catch (AWTException e) {
99
+	    e.printStackTrace();
100
+	}
101
+
102
+	for(Key k: parser.parse("W-r"))
103
+	    if(k.isPress())
104
+		r.keyPress(k.getCode());
105
+	    else
106
+		r.keyRelease(k.getCode());
107
+
108
+    }
109
+}

+ 68
- 0
project/src/me/sciion/agent/utils/parser/KeyScanner.java View File

@@ -0,0 +1,68 @@
1
+package me.sciion.agent.utils.parser;
2
+
3
+import java.util.LinkedList;
4
+import java.util.Stack;
5
+
6
+public class KeyScanner {
7
+
8
+    
9
+    private Stack<Token> tokens;
10
+    private LinkedList<Character> input;
11
+    public KeyScanner(String line){
12
+	tokens = new Stack<Token>();
13
+	input = new LinkedList<Character>();
14
+	for(int i = 0; i < line.length(); i++){
15
+	    input.add(line.charAt(i));
16
+	}
17
+    }
18
+    
19
+    private char scan(){
20
+	if(input.isEmpty()){
21
+	    return '\0';
22
+	}
23
+	char c = input.pollFirst();
24
+	return c;
25
+    }
26
+    
27
+    private void putBack(char c){
28
+	input.addFirst(c);
29
+    }
30
+    
31
+    public Token next(){
32
+	
33
+	int state = 0;
34
+	    char c ='\0';
35
+	while(true){
36
+	
37
+	    switch(state){
38
+	    	case 0: 
39
+		    c = scan();
40
+		    if(c == '\0')
41
+			return null;
42
+		    else if(c == 'M')
43
+	    		state = 1;
44
+	    	    else if(c == 'W')
45
+	    		state = 1;
46
+	    	    else if(c == 'S')
47
+	    		state = 1;
48
+	    	    else if(c == 'C')
49
+	    		state = 1;
50
+	    	    else
51
+	    		return new LiteralToken(c);
52
+		break;
53
+	    	case 1: 
54
+	    	    char c2 = scan();
55
+	    	    if(c2 == '-')
56
+	    		return new ModifierToken(c);
57
+	    	    else{
58
+	    		putBack(c2);
59
+	    		state = 0;
60
+	    		return new LiteralToken(c);
61
+	    	    }
62
+		default:
63
+		    System.out.println("Error in scanner. Current char: " + c);
64
+		    return null;
65
+	    }
66
+	}
67
+    }
68
+}

+ 14
- 0
project/src/me/sciion/agent/utils/parser/LiteralToken.java View File

@@ -0,0 +1,14 @@
1
+package me.sciion.agent.utils.parser;
2
+
3
+public class LiteralToken extends Token{
4
+
5
+    public LiteralToken(char value) {
6
+	super(value);
7
+    }
8
+
9
+    @Override
10
+    public TokenType getType() {
11
+	return TokenType.LITERAL;
12
+    }
13
+
14
+}

+ 14
- 0
project/src/me/sciion/agent/utils/parser/ModifierToken.java View File

@@ -0,0 +1,14 @@
1
+package me.sciion.agent.utils.parser;
2
+
3
+public class ModifierToken extends Token{
4
+
5
+    public ModifierToken(char value) {
6
+	super(value);
7
+    }
8
+
9
+    @Override
10
+    public TokenType getType() {
11
+	return TokenType.MODIFIER;
12
+    }
13
+
14
+}

+ 17
- 0
project/src/me/sciion/agent/utils/parser/Token.java View File

@@ -0,0 +1,17 @@
1
+package me.sciion.agent.utils.parser;
2
+
3
+public abstract class Token {
4
+
5
+    
6
+    private char value;
7
+    public Token(char value){
8
+	this.value = value;
9
+    }
10
+    
11
+    public char getValue(){
12
+	return value;
13
+    }
14
+    
15
+    public abstract TokenType getType();
16
+    
17
+}

+ 6
- 0
project/src/me/sciion/agent/utils/parser/TokenType.java View File

@@ -0,0 +1,6 @@
1
+package me.sciion.agent.utils.parser;
2
+
3
+public enum TokenType {
4
+
5
+    LITERAL, MODIFIER
6
+}

+ 14
- 0
scripts/HelloAdnan.py View File

@@ -0,0 +1,14 @@
1
+import sys
2
+sys.path.append("lib/AgentAPI.jar")
3
+from me.sciion.agent.api import API
4
+template1 = API.loadTemplate("assets/templates/template-sublime.png")
5
+API.move(API.locate(template1))
6
+API.click()
7
+API.type("C-aC-c")
8
+
9
+template2 = API.loadTemplate("assets/templates/template-firefox.png")
10
+API.move(API.locate(template2))
11
+API.click()
12
+
13
+template2 = API.loadTemplate("assets/templates/Template-Adnane.png")
14
+API.move(API.locate(template3))

+ 7
- 0
scripts/TypingExample.py View File

@@ -0,0 +1,7 @@
1
+import sys
2
+sys.path.append("lib/AgentAPI.jar")
3
+from me.sciion.agent.api import API
4
+API.type("W-rnotepad")
5
+API.type("Hello World")
6
+API.type("AltS-.")
7
+API.type("S-hello S-world S-7S-7 GUI-Agent")

Loading…
Cancel
Save