Clojush-Tools/csv_success_by_program_executions.py at master · thelmuth/Clojush-Tools · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
#!/usr/bin/python

"""
For each success, records the program executions along with the number of train, test, and simp_test solutions at that point.
"""

import os, sys
from sys import maxint

# Set these before running:
problem = "vector-average"
error_threshold = 0

max_executions = 75000000

normal_training_cases_size = int(max_executions / (1000 * 300))
print "Normal training cases size =", normal_training_cases_size

# Dirs for downsampled lexicase
# dirs = [("lexicase", normal_training_cases_size, "/home/thelmuth/Results/parent-selection-v3-UMAD/lexicase/%s" % problem),
#         ("downsample-25", int(normal_training_cases_size * 0.25), "/home/thelmuth/Results/parent-selection-v3-UMAD/downsample-lexicase/rate-0.25/%s" % problem),
#         ("downsample-10", int(normal_training_cases_size * 0.1), "/home/thelmuth/Results/parent-selection-v3-UMAD/downsample-lexicase/rate-0.1/%s" % problem)
# ]


dirs = [#("lexicase", normal_training_cases_size, "/home/thelmuth/Results/parent-selection-v3-UMAD/lexicase/%s" % problem),
        ("CDGP", False, "/home/thelmuth/Collab/thelmuth/Results/counterexample-driven-gp/no-generational-case-additions/%s" % problem),
        ("CDGPq80", False, "/home/thelmuth/Collab/thelmuth/Results/counterexample-driven-gp/cases-passed-threshold-0.8/%s" % problem),
        ("CDGP50", False, "/home/thelmuth/Collab/thelmuth/Results/counterexample-driven-gp/add-case-after-50-gens/%s" % problem)
]

# dirs = [#("lexicase", normal_training_cases_size, "/home/thelmuth/Results/parent-selection-v3-UMAD/lexicase/%s" % problem),
#         ("CDGP", False, "/home/thelmuth/Collab/thelmuth/Results/counterexample-driven-gp/no-generational-case-additions/%s" % problem),
#         ("CDGP50", False, "/home/thelmuth/Collab/thelmuth/Results/counterexample-driven-gp/add-case-after-50-gens/%s" % problem)
# ]


for (prob, rate, direct) in dirs:
    print prob, " - ", direct


outputFilePrefix = "log"
outputFileSuffix = ".txt"


def getSuccessProgExecs(outputDirectory, cases_size):
    # Main area
    i = 0

    if outputDirectory[-1] != '/':
        outputDirectory += '/'
    dirList = os.listdir(outputDirectory)

    train_success_progexecs = []
    test_success_progexecs = []
    simplified_test_success_progexecs = []

    while (outputFilePrefix + str(i) + outputFileSuffix) in dirList:

        runs = i + 1 # After this loop ends, runs should be correct
        fileName = (outputFilePrefix + str(i) + outputFileSuffix)
        f = open(outputDirectory + fileName)

        prog_execs = -1
        trainSuccess = -1
        testSuccess = -1
        simpTestSuccess = -1

        for line in reversed(f.readlines()):

            if line.startswith("Test total error for best:") and simpTestSuccess == -1:
                try:
                    simpBestTest = int(line.split()[-1].strip("Nn"))
                except ValueError, e:
                    simpBestTest = float(line.split()[-1].strip("Nn"))
                if simpBestTest <= error_threshold:
                    simpTestSuccess = True
                else:
                    simpTestSuccess = False

            elif line.startswith("Test total error for best:") and simpTestSuccess != -1:
                try:
                    bestTest = int(line.split()[-1].strip("Nn"))
                except ValueError, e:
                    bestTest = float(line.split()[-1].strip("Nn"))
                if bestTest <= error_threshold:
                    testSuccess = True
                else:
                    testSuccess = False

            if line.startswith("SUCCESS"):
                trainSuccess = True
            if line.startswith("FAILURE"):
                trainSuccess = False
                testSuccess = False
                simpTestSuccess = False

            if line.startswith("Number of program evaluations used so far:") and prog_execs == -1:
                prog_execs = int(line.split()[-1]) * cases_size

            if line.startswith("Number of program executions (running on a single case counts as 1 execution):") and prog_execs == -1:
                prog_execs = int(line.split()[-1])

            if prog_execs >= 0 and trainSuccess != -1 and testSuccess != -1 and simpTestSuccess != -1:
                break

        if trainSuccess == True: # Might be -1, which would need to be False here
            train_success_progexecs.append(prog_execs)
        if testSuccess == True: # Might be -1, which would need to be False here
            test_success_progexecs.append(prog_execs)
        if simpTestSuccess == True: # Might be -1, which would need to be False here
            simplified_test_success_progexecs.append(prog_execs)

        i += 1


    return (train_success_progexecs, test_success_progexecs, simplified_test_success_progexecs)

def numbers_leq_x(nums, x):
    """Returns the count of the number of numbers <= x in list nums"""
    return len(filter(lambda a: a <= x, nums))

#### Printing

print "method,program_executions,train,test,simp"

for (prob, cases_size, directory) in dirs:
    (train_success_progexecs, test_success_progexecs, simplified_test_success_progexecs) = getSuccessProgExecs(directory, cases_size)

    train_success_progexecs.sort()
    test_success_progexecs.sort()
    simplified_test_success_progexecs.sort()

    print "%s,0,0,0,0" % prob

    for prog_execs in train_success_progexecs:
        train = numbers_leq_x(train_success_progexecs, prog_execs)
        test =  numbers_leq_x(test_success_progexecs, prog_execs)
        simp =  numbers_leq_x(simplified_test_success_progexecs, prog_execs)

        print "%s,%i,%i,%i,%i" % (prob, prog_execs, train, test, simp)

    print "%s,%i,%i,%i,%i" % (prob, max_executions, train, test, simp)