Browse Source

added support for states, added som states, expanded on the examples and removed the enter from API:type()

master
sciion 3 years ago
parent
commit
cf31a5dd24
No account linked to committer's email address

BIN
lib/AgentAPI.jar View File


+ 1
- 1
project/src/me/sciion/agent/Run.java View File

@@ -37,7 +37,7 @@ public class Run {
37 37
 		screenMat.put(0, 0, pixels);
38 38
 		return screenMat;
39 39
 	}
40
-
40
+	
41 41
 	public static BufferedImage cvtBufferedImage(Mat rawIn, int type) {
42 42
 		Mat raw = new Mat();
43 43
 		rawIn.copyTo(raw);

+ 61
- 7
project/src/me/sciion/agent/api/API.java View File

@@ -4,15 +4,21 @@ import java.awt.image.BufferedImage;
4 4
 import java.awt.image.DataBufferByte;
5 5
 import java.util.Arrays;
6 6
 
7
+import javax.swing.JOptionPane;
8
+
7 9
 import org.opencv.core.Core;
8 10
 import org.opencv.core.CvType;
9 11
 import org.opencv.imgproc.Imgproc;
12
+import org.w3c.dom.Document;
13
+import org.w3c.dom.Node;
14
+import org.w3c.dom.NodeList;
10 15
 
11 16
 import me.sciion.agent.Run;
12 17
 import me.sciion.agent.environments.HostSystem;
13 18
 import me.sciion.agent.environments.VirtualEnvironment;
14 19
 import me.sciion.agent.matching.Match;
15 20
 import me.sciion.agent.matching.Matcher;
21
+import me.sciion.agent.state.State;
16 22
 import me.sciion.agent.state.Template;
17 23
 import me.sciion.agent.utils.Location;
18 24
 
@@ -33,22 +39,63 @@ public class API {
33 39
 
34 40
     public static Template loadTemplate(String path) {
35 41
 
36
-	BufferedImage templateImage = Run.getImageResource(path);
37
-	return new Template("Undefined template id", Imgproc.TM_CCOEFF_NORMED, 0.97, path, templateImage);
42
+	if (path.contains(".png")) {
43
+	    BufferedImage templateImage = Run.getImageResource(path);
44
+	    return new Template("Undefined template id", Imgproc.TM_CCOEFF_NORMED, 0.97, path, templateImage);
45
+	} else if (path.contains(".xml")) {
46
+	    Document doc = Run.parseFromXML(path);
47
+	    Node n = doc.getElementsByTagName("template").item(0);
48
+	    if (n.getNodeType() != Node.ELEMENT_NODE)
49
+		return null;
50
+	    Template t = Template.fromXML(n);
51
+	    return t;
52
+	}
53
+	else{
54
+	    System.err.println("Unrecognized file ending \"" + path + "\"");
55
+	    return null;
56
+	}
57
+
58
+    }
59
+
60
+    public static State loadState(String path) {
61
+	 if (path.contains(".xml")) {
62
+		Document doc = Run.parseFromXML(path);
63
+		Node n = doc.getElementsByTagName("state").item(0);
64
+		if (n.getNodeType() != Node.ELEMENT_NODE)
65
+		    return null;
66
+		State s = State.fromXML(n);
67
+		return s;
68
+	 }
69
+	 else{
70
+	     System.err.println("Could not load state from: " + path);
71
+	     return null;
72
+	 }
73
+
38 74
     }
39 75
 
40 76
     public static boolean exists(Template template) {
41 77
 	Match m = matcher.match(Run.cvtMat(environment.getScreenshot(), CvType.CV_8UC3), template);
42 78
 	return m.getScore() >= template.getThreshold();
43 79
     }
44
-
80
+    
81
+    public static boolean exists(State state) {
82
+	boolean exists = true;
83
+	for(Template t: state.getTemplates()){
84
+		Match m = matcher.match(Run.cvtMat(environment.getScreenshot(), CvType.CV_8UC3), t);
85
+		if(m.getScore() < t.getThreshold()){
86
+		    exists = false;
87
+		}
88
+	}
89
+	return exists;
90
+    }
91
+    
45 92
     public static void waitForChange() {
46 93
 	int hash = 0;
47 94
 	int hash2 = -1;
48 95
 	BufferedImage first = environment.getScreenshot();
49 96
 	do {
50 97
 	    try {
51
-		Thread.sleep(1);
98
+		Thread.sleep(10);
52 99
 	    } catch (InterruptedException e) {
53 100
 		e.printStackTrace();
54 101
 	    }
@@ -57,7 +104,6 @@ public class API {
57 104
 	    hash2 = Arrays.hashCode(((DataBufferByte) second.getRaster().getDataBuffer()).getData());
58 105
 	} while (hash == hash2);
59 106
 	return;
60
-
61 107
     }
62 108
 
63 109
     public static boolean waitForTemplate(Template template) {
@@ -72,7 +118,14 @@ public class API {
72 118
 	} while (m.getScore() < template.getThreshold());
73 119
 	return true;
74 120
     }
75
-
121
+    
122
+    public static boolean waitForState(State state){
123
+	for(Template t: state.getTemplates()){
124
+	    waitForTemplate(t);
125
+	}
126
+	return true;
127
+    }
128
+    
76 129
     public static Location locate(Template template) {
77 130
 	Match m = matcher.match(Run.cvtMat(environment.getScreenshot(), CvType.CV_8UC3), template);
78 131
 	return new Location(m.getX() + template.getImage().getWidth() / 2,
@@ -94,7 +147,8 @@ public class API {
94 147
     public static String getVersion() {
95 148
 	return VERSION;
96 149
     }
97
-    public static void main(String [] args){
150
+
151
+    public static void main(String[] args) {
98 152
 	
99 153
     }
100 154
 }

+ 3
- 3
project/src/me/sciion/agent/environments/HostSystem.java View File

@@ -23,7 +23,7 @@ public class HostSystem implements VirtualEnvironment{
23 23
     public HostSystem() {
24 24
 	try {
25 25
 	    robot = new Robot();
26
-	    robot.setAutoDelay(50);
26
+	    robot.setAutoDelay(1);
27 27
 	} catch (AWTException e) {
28 28
 	    e.printStackTrace();
29 29
 	}
@@ -50,8 +50,8 @@ public class HostSystem implements VirtualEnvironment{
50 50
 	    else
51 51
 		robot.keyRelease(k.getCode());
52 52
 	}
53
-	robot.keyPress(KeyEvent.VK_ENTER);
54
-	robot.keyRelease(KeyEvent.VK_ENTER);
53
+	//robot.keyPress(KeyEvent.VK_ENTER);
54
+	//robot.keyRelease(KeyEvent.VK_ENTER);
55 55
 
56 56
 }
57 57
 

+ 8
- 1
project/src/me/sciion/agent/state/State.java View File

@@ -47,7 +47,14 @@ public class State {
47 47
 	public void setId(String id) {
48 48
 		this.id = id;
49 49
 	}
50
-
50
+	
51
+	public Template getTemplate(String name){
52
+	    for(Template t: templates)
53
+		if(t.getId().equals(name))
54
+		    return  t;
55
+	    return null;
56
+	}
57
+	
51 58
 	public Vector<Template> getTemplates() {
52 59
 		return templates;
53 60
 	}

+ 12
- 0
scripts/RunStateExample.py View File

@@ -0,0 +1,12 @@
1
+import sys
2
+sys.path.append("lib/AgentAPI.jar")
3
+from me.sciion.agent.api import API
4
+run_state = API.loadState("states/Run.xml")
5
+API.type("W-r")
6
+API.waitForState(run_state)
7
+API.type("notepad")
8
+location = API.locate(run_state.getTemplate("ok"))
9
+API.move(location)
10
+API.click()
11
+API.waitForChange()
12
+API.type("Hello WorldS-1")

+ 8
- 0
scripts/StateExample.py View File

@@ -0,0 +1,8 @@
1
+import sys
2
+sys.path.append("lib/AgentAPI.jar")
3
+from me.sciion.agent.api import API
4
+state1 = API.loadState("states/TaskBar.xml")
5
+if API.exists(state1):
6
+	print "Found state ", state1
7
+else:
8
+	print state1, " not found."

+ 5
- 0
states/Run.xml View File

@@ -0,0 +1,5 @@
1
+<state name="Run">
2
+	<template id="run-icon" method="TM_CCOEFF_NORMED" path="states/templates-Run/run-icon.png" threshold="0.97"/>
3
+	<template id="browse" method="TM_CCOEFF_NORMED" path="states/templates-Run/browse.png" threshold="0.97"/>
4
+	<template id="ok" method="TM_CCOEFF_NORMED" path="states/templates-Run/ok.png" threshold="0.5"/>
5
+</state>

+ 4
- 0
states/TaskBar.xml View File

@@ -0,0 +1,4 @@
1
+<state name="Task bar windows 10">
2
+	<template id="start-menu" method="TM_CCOEFF_NORMED" path="states/templates-TaskBar/start-menu-icon.png" threshold="0.97"/>
3
+	<template id="search-and-workspaces" method="TM_CCOEFF_NORMED" path="states/templates-TaskBar/search-and-workspaces-icons.png" threshold="0.97"/>
4
+</state>

BIN
states/templates-Run/browse.png View File


BIN
states/templates-Run/ok.png View File


BIN
states/templates-Run/run-icon.png View File


BIN
states/templates-TaskBar/search-and-workspaces-icons.png View File


BIN
states/templates-TaskBar/start-menu-icon.png View File


Loading…
Cancel
Save