Difference between revisions of "Tutorial Moodle VPL Tips & Tricks"
(→Generating a vpl_evaluate.cases File) |
(→How to Differentiate Between Run and Evaluate Modes?) |
||
(21 intermediate revisions by the same user not shown) | |||
Line 9: | Line 9: | ||
[[Image:MoodleVPLLogo.png]] | [[Image:MoodleVPLLogo.png]] | ||
|} | |} | ||
+ | <br /> | ||
+ | ---- | ||
+ | <center>[[Tutorials#Moodle_Virtual-Programming-Lab_.28VPL.29|Moodle VPL Tutorials]]</center> | ||
+ | ---- | ||
+ | <br /> | ||
<br /> | <br /> | ||
<bluebox> | <bluebox> | ||
Line 16: | Line 21: | ||
=Tips & Tricks= | =Tips & Tricks= | ||
− | ==Output of Evaluate Step== | + | ==How to Differentiate Between Run and Evaluate Modes?== |
+ | <br /> | ||
+ | This solution was provided to me by Sashini Herath, of CS Department at the University of Moratuwa, in Sri Lanka (http://cse.mrt.ac.lk), with insights from Juan Carlos Rodríguez-del-Pino. Please see [https://moodle.org/mod/forum/discuss.php?d=154988&parent=1411441 https://moodle.org/mod/forum/discuss.php?d=154988&parent=1411441] for reference. | ||
+ | <br /> | ||
+ | ===vpl_run.sh=== | ||
+ | <br /> | ||
+ | ::<source lang="bash"> | ||
+ | #!/bin/bash | ||
+ | echo "#!/bin/bash" > vpl_execution | ||
+ | if [ -f ./vpl_evaluate.sh ] ; then | ||
+ | echo "python2.7 evaluateProgram.py" >> vpl_execution | ||
+ | else | ||
+ | echo "python2.7 runProgram.py" >> vpl_execution | ||
+ | fi | ||
+ | chmod +x vpl_execution | ||
+ | </source> | ||
+ | <br /> | ||
+ | ===runProgram.py=== | ||
+ | <br /> | ||
+ | ::<source lang="python"> | ||
+ | # runProgram.py | ||
+ | # students, do not modify this file. It will be used to test your program. | ||
+ | |||
+ | import sys | ||
+ | |||
+ | #--- run the program to test --- | ||
+ | import Lab_9_Additional | ||
+ | |||
+ | #--- display the contents of the sample.txt file which should | ||
+ | #--- have been modified by the program under test. | ||
+ | |||
+ | print "\n---Output.txt---" | ||
+ | try: | ||
+ | file = open( "Output.txt", "r" ) | ||
+ | print file.read(), | ||
+ | file.close() | ||
+ | except IOError as e: | ||
+ | print "I/O error({0}): {1}".format(e.errno, e.strerror) | ||
+ | except: | ||
+ | print "Unexpected error:", sys.exc_info()[0] | ||
+ | raise | ||
+ | </source> | ||
+ | <br /> | ||
+ | ===evaluateProgram.py=== | ||
+ | <br /> | ||
+ | ::<source lang="python"> | ||
+ | # evaluateProgram.py | ||
+ | # students, do not modify this file. It will be used to test your program. | ||
+ | |||
+ | import sys | ||
+ | |||
+ | #--- capture the stdout of the program to test into a file | ||
+ | saveStdOut = sys.stdout | ||
+ | sys.stdout = open( "fakestdout.txt", "w" ) | ||
+ | |||
+ | #--- run the program to test --- | ||
+ | import Lab_9_Additional | ||
+ | |||
+ | #--- get rid of whatever it printed on the stdout --- | ||
+ | sys.stdout = saveStdOut | ||
+ | |||
+ | #--- display the contents of the sample.txt file which should | ||
+ | #--- have been modified by the program under test. | ||
+ | |||
+ | try: | ||
+ | file = open( "Output.txt", "r" ) | ||
+ | print file.read(), | ||
+ | file.close() | ||
+ | except IOError as e: | ||
+ | print "I/O error({0}): {1}".format(e.errno, e.strerror) | ||
+ | except: | ||
+ | print "Unexpected error:", sys.exc_info()[0] | ||
+ | raise | ||
+ | </source> | ||
+ | <br /> | ||
+ | |||
+ | ==Bash Quick Solutions== | ||
+ | <br /> | ||
+ | ; To compare the student's output to the expected output, but not worry about order of lines output, or whitespace: | ||
+ | <br /> | ||
+ | ::<source lang="bash"> | ||
+ | # code included in vpl_evaluate.sh. All $-signs prefixed with back-slash. | ||
+ | # $user contains output from student program | ||
+ | # $expected contains expected output | ||
+ | cat \$user | sort | xargs > \${user}_sorted | ||
+ | cat \$expected | sort | xargs > \${expected}_sorted | ||
+ | diff -y -w -B --ignore-all-space \${user}_sorted \${expected}_sorted > diff.out | ||
+ | |||
+ | if ((\$? > 0)); then | ||
+ | echo "Comment :=>> Your output is incorrect." | ||
+ | echo "Comment :=>> ---------------" | ||
+ | echo "Comment :=>>- Your output:" | ||
+ | echo "Comment :=>> ---------------" | ||
+ | echo "<|--" | ||
+ | cat \$user | ||
+ | echo "--|>" | ||
+ | echo "" | ||
+ | echo "Comment :=>> ---------------" | ||
+ | echo "Comment :=>>- Expected output " | ||
+ | echo "Comment :=>> ---------------" | ||
+ | echo "<|--" | ||
+ | cat \$expected | ||
+ | echo "--|>" | ||
+ | else | ||
+ | echo "Comment :=>> Correct output." | ||
+ | noTestsPassed=\$((noTestsPassed+1)) | ||
+ | fi | ||
+ | </source> | ||
+ | <br /> | ||
+ | ; To remove comments from a '''Java''' program, including /* ... */, /** ... */, and //: | ||
+ | <br /> | ||
+ | ::<source lang="bash"> | ||
+ | |||
+ | prog1=yourprogramname # the name of the java code, '''without''' the ".java" extension | ||
+ | cat $prog1.java | sed 's://.*$::g' | sed '/\/\*\*/,/\*\// {s/.*\*\/.*//p; d}' > _$prog1.java | ||
+ | </source> | ||
+ | <br /> | ||
+ | ; To remove the comments from an assembly program (.asm): | ||
+ | <br /> | ||
+ | ::<source lang="bash"> | ||
+ | prog1=yourprogramname # then name of the asm code, '''without''' the ".asm" extension | ||
+ | cat ${prog1}.asm | sed 's:;.*$::g' > _${prog1}.asm | ||
+ | </source> | ||
+ | <br /> | ||
+ | ; To stop programs with endless loops, use the '''timeout''' command. | ||
+ | <br /> | ||
+ | :(Note: the \$ are required inside vpl_evaluate.sh when creating vpl_execute.sh) | ||
+ | ::<source lang="bash"> | ||
+ | timeout 30 java \$prog1 < dataIn.txt &> user.out | ||
+ | retcode=\$? | ||
+ | |||
+ | #--- if we killed the waiter, everything is good --- | ||
+ | if [ \$retcode -eq 124 ]; then | ||
+ | echo "Comment :=>>-Your program has stopped working." | ||
+ | else | ||
+ | echo "Comment :=>>-Your program ran to completion." | ||
+ | fi | ||
+ | </source> | ||
+ | <br /> | ||
+ | ; To add @suppresswarnings() to Java programs that don't contain it: | ||
+ | <br /> | ||
+ | <source lang="bash"> | ||
+ | addSuppressWarnings() { | ||
+ | if [ "\$1" ]; then # if there's a parameter | ||
+ | prog=\$1 | ||
+ | if grep SuppressWarnings $prog ; then | ||
+ | : | ||
+ | else | ||
+ | #echo "Not found" | ||
+ | sed -i 's/^class/@SuppressWarnings("unchecked")\nclass/' \${prog} | ||
+ | sed -i 's/^public class/@SuppressWarnings("unchecked")\npublic class/' \${prog} | ||
+ | fi | ||
+ | fi | ||
+ | } | ||
+ | |||
+ | ###### (and in the code of ''vpl_evaluate.sh'', call function this way: ###### | ||
+ | # --- add a @SuppressWarnings( "unchecked" ) label | ||
+ | addSuppressWarnings( \${prog1}.java ) | ||
+ | </source> | ||
+ | <br /> | ||
+ | |||
+ | ==Output of ''Evaluate'' Step== | ||
* The output of the Evaluate action is a bit illogical: | * The output of the Evaluate action is a bit illogical: | ||
Line 43: | Line 209: | ||
<br /> | <br /> | ||
− | ==Generating a vpl_evaluate.cases File== | + | |
+ | ==Generating a vpl_evaluate.cases File Quickly== | ||
<br /> | <br /> | ||
;Recommendations | ;Recommendations | ||
Line 52: | Line 219: | ||
# You should be ready to evaluate the solution program and get 100/100 as a grade. | # You should be ready to evaluate the solution program and get 100/100 as a grade. | ||
<br /> | <br /> | ||
+ | ==Creating a ''vpl_execution'' script that measures the size of a program== | ||
+ | <br /> | ||
+ | Imagine that the assignment for the student is to create a program that solves a particular problem (which can be tested separately for the correctness of its output), but the requirement is to make the program executable as small as possible. | ||
+ | The grade given to the submitted program will be 100 if the size is equal to the size of the solution program, and some lower grade depending on how large the file is. | ||
+ | <br /> | ||
+ | Here's a '''bash''' script that will do the trick | ||
+ | <br /> | ||
+ | :<source lang="bash"> | ||
+ | #! /bin/bash | ||
+ | # $file contains the name of the executable. The executable | ||
+ | # is created by assembling ${file}.asm, generating ${file}.o, | ||
+ | # and linking ${file}.o. | ||
+ | # The script outputs 0 if the executable crashes, or just | ||
+ | # an integer that is the size of the executable, in bytes, if | ||
+ | # it runs correctly. | ||
+ | |||
+ | file="printStars" # whatever executable is to be tested | ||
+ | |||
+ | # assemble and link the executable | ||
+ | nasm -f elf -F stabs ${file}.asm | ||
+ | ld -melf_i386 -o $file ${file}.o | ||
+ | |||
+ | # run the executable in a subshell to catch any segmentation fault error | ||
+ | # that may result. | ||
+ | { ./$file &> /dev/null ; } &> log | ||
+ | |||
+ | # if an error occured, automatically return 0 | ||
+ | if [ $? -eq 139 ]; then | ||
+ | echo "0"; | ||
+ | exit 0; | ||
+ | fi | ||
+ | |||
+ | # print out the size of the file | ||
+ | size=$(stat -c%s "$file") | ||
+ | echo $size | ||
+ | </source> | ||
+ | <br /> | ||
+ | The part of the script that reads <tt>{ ./$file &> /dev/null ; } &> log</tt> is a bit complex. The reason for this is that we want to catch a potential crash by the program being tested, prevent it from outputting some ''segmentation fault'' message that would have come up, and automatically output 0 for it, and not its size. | ||
+ | |||
+ | If the program runs correctly, its output is discarded and the script simply outputs the size in bytes of the executable. | ||
+ | <br /> | ||
+ | ==Creating a ''vpl_execution'' Script that Outputs the Grade a Program Based on its Size== | ||
+ | <br /> | ||
+ | This time we take the previous version of '''vpl_execution''' and make it output the grade (0 to 100) based on how small the program executable is. The smallest possible size is assumed to be known. | ||
+ | <br /> | ||
+ | :<source lang="bash"> | ||
+ | #! /bin/bash | ||
+ | # $file contains the name of the executable. The executable | ||
+ | # is created by assembling ${file}.asm, generating ${file}.o, | ||
+ | # and linking ${file}.o. | ||
+ | # The script outputs 0 if the executable crashes, or just | ||
+ | # an integer that is the size of the executable, in bytes, if | ||
+ | # it runs correctly. | ||
+ | |||
+ | file="printStars" # whatever executable is to be tested | ||
+ | bestSize=1065 # the size of the executable version of the solution program | ||
+ | # in bytes. | ||
+ | |||
+ | # assemble and link the executable | ||
+ | nasm -f elf -F stabs ${file}.asm | ||
+ | ld -melf_i386 -o $file ${file}.o | ||
+ | |||
+ | # run the executable in a subshell to catch any segmentation fault error | ||
+ | # that may result. | ||
+ | { ./$file &> /dev/null ; } &> log | ||
+ | |||
+ | # if an error occured, automatically return 0 | ||
+ | if [ $? -eq 139 ]; then | ||
+ | echo "0"; | ||
+ | exit 0; | ||
+ | fi | ||
+ | |||
+ | # print out the size of the file | ||
+ | size=$(stat -c%s "$file") | ||
+ | #echo $size | ||
+ | |||
+ | # compare size to best size recorded and output | ||
+ | # grade based on size. | ||
+ | |||
+ | increment=20 # size increment that takes away 10 more points | ||
+ | # from the grade | ||
+ | |||
+ | # possible grades | ||
+ | grades=( 100 90 80 70 60 50 40 30 20 10 0 ) | ||
+ | |||
+ | # find bin corresponding to size of executable, based on | ||
+ | # $bestSize, and bins of width $increment | ||
+ | for i in {0..10} ; do | ||
+ | bestSize=$(($bestSize+$increment)) | ||
+ | if [ $size -le $bestSize ] ; then | ||
+ | echo ${grades[$i]} | ||
+ | exit 0 | ||
+ | fi | ||
+ | done | ||
+ | echo "0" | ||
+ | </source> | ||
+ | <!-- ---------------------------------------------------------------------------------------------------------------- --> | ||
+ | <br /> | ||
+ | |||
+ | =Useful Bash Functions for vpl_evaluate.sh= | ||
+ | <br /> | ||
+ | ==Function to remove non numbers from a file == | ||
+ | <br /> | ||
+ | <source lang="bash"> | ||
+ | # function that removes non-digits, extra spaces, and extra blank lines from text file. | ||
+ | # parameter is the name of a text file (typically output of user program, or output of | ||
+ | # solution program). | ||
+ | cleanup () { | ||
+ | if [ "\$1" ]; then # if there's a parameter | ||
+ | |||
+ | #--- remove non numbers and non minus--- | ||
+ | cat \$1 | sed 's/[^0-9*.0-9\ ]*//g' > dummy.out | ||
+ | cp dummy.out \$1 | ||
+ | |||
+ | #--- remove multiple spaces --- | ||
+ | cat \$1 | sed 's/ */ /g' > dummy.out | ||
+ | cp dummy.out \$1 | ||
+ | |||
+ | cat \$1 | sed 's/^[ \t]*//' > dummy.out | ||
+ | cp dummy.out \$1 | ||
+ | |||
+ | #--- remove blank lines --- | ||
+ | cat \$1 | sed '/^\s*\$/d' > dummy.out | ||
+ | cp dummy.out \$1 | ||
+ | fi | ||
+ | } | ||
+ | |||
+ | </source> | ||
<br /> | <br /> | ||
<br /> | <br /> | ||
+ | == Report discrepancy between user and solution output == | ||
<br /> | <br /> | ||
+ | <source lang="bash"> | ||
+ | # function that prints the difference between user and expected output | ||
+ | # \$TEMP is a parameter used to test | ||
+ | incorrectOutput() { | ||
+ | echo "Comment :=>>- Your output is incorrect." | ||
+ | #--- display test file --- | ||
+ | echo "<|--" | ||
+ | echo "Your program tested with \$TEMP" | ||
+ | echo "--|>" | ||
+ | |||
+ | echo "Comment :=>> ---------------" | ||
+ | echo "Comment :=>>- Your output:" | ||
+ | echo "Comment :=>> ---------------" | ||
+ | echo "<|--" | ||
+ | cat userOut.org | ||
+ | echo "--|>" | ||
+ | echo "" | ||
+ | echo "Comment :=>> ---------------" | ||
+ | echo "Comment :=>>- Expected output: " | ||
+ | echo "Comment :=>> ---------------" | ||
+ | echo "<|--" | ||
+ | cat expectedOut.org | ||
+ | echo "--|>" | ||
+ | |||
+ | } | ||
+ | |||
+ | </source> | ||
<br /> | <br /> | ||
<br /> | <br /> | ||
+ | == Loop and Test Program with 3 Different Parameters == | ||
<br /> | <br /> | ||
+ | <source lang="bash"> | ||
+ | for TEMP in \$TEMP1 \$TEMP2 \$TEMP3 ; do | ||
+ | # create input for the program | ||
+ | echo \$TEMP > input | ||
+ | |||
+ | # ================================================= | ||
+ | # generate user output and expected output | ||
+ | \$python \$prog < input > userOut | ||
+ | \$python \$solutionProg < input > expectedOut | ||
+ | |||
+ | cp userOut userOut.org | ||
+ | cp expectedOut expectedOut.org | ||
+ | |||
+ | cleanup userOut | ||
+ | cleanup expectedOut | ||
+ | |||
+ | #--- compute difference --- | ||
+ | diff -y -w --ignore-all-space userOut expectedOut > diff.out | ||
+ | |||
+ | #--- reject if different --- | ||
+ | if ((\$? > 0)); then | ||
+ | incorrectOutput | ||
+ | grade=65 | ||
+ | # --------------------- REWARD IF CORRECT OUTPUT ----------------- | ||
+ | else | ||
+ | #--- good output --- | ||
+ | echo "Comment :=>>- Congrats, your output is correct." | ||
+ | echo "Comment :=>> --------------------------------." | ||
+ | echo "<|--" | ||
+ | echo "Your program tested with \$TEMP" | ||
+ | echo "" | ||
+ | cat userOut | ||
+ | echo "--|>" | ||
+ | grade=100 | ||
+ | fi | ||
+ | |||
+ | done | ||
+ | </source> | ||
<br /> | <br /> | ||
+ | <br /> | ||
+ | == Report Grade and Max it to 100 if Above == | ||
+ | <br /> | ||
+ | <source lang="bash"> | ||
+ | # Report grade | ||
+ | if (( grade > 100 )); then | ||
+ | grade=100 | ||
+ | fi | ||
+ | echo "Grade :=>> \$grade" | ||
+ | |||
+ | |||
+ | </source> | ||
+ | <br /> | ||
+ | <br /> | ||
+ | == Pick 3 Different Random Inputs Test Values == | ||
+ | <br /> | ||
+ | <source lang="bash"> | ||
+ | # ================================================= | ||
+ | # Pick 3 random inputs to test program with | ||
+ | TEMP1=\$RANDOM | ||
+ | let "TEMP1 %= 30" | ||
+ | let "TEMP1 = TEMP1 + 6" | ||
+ | TEMP2=\$RANDOM | ||
+ | let "TEMP2 %= 30" | ||
+ | let "TEMP2 = 0 - TEMP2" | ||
+ | TEMP3=\$RANDOM | ||
+ | let "TEMP3 %= 30" | ||
+ | let "TEMP3 = TEMP3 + 2" | ||
+ | TEMP3=\${TEMP3}.5 | ||
+ | |||
+ | </source> | ||
+ | <br /> | ||
+ | |||
<br /> | <br /> | ||
<br /> | <br /> |
Latest revision as of 11:00, 30 March 2017
--D. Thiebaut (talk) 20:31, 11 June 2014 (EDT)
This is a collection of tips, tricks, observations, recommendations, and modifications related to using VPL with Moodle.
Tips & Tricks
How to Differentiate Between Run and Evaluate Modes?
This solution was provided to me by Sashini Herath, of CS Department at the University of Moratuwa, in Sri Lanka (http://cse.mrt.ac.lk), with insights from Juan Carlos Rodríguez-del-Pino. Please see https://moodle.org/mod/forum/discuss.php?d=154988&parent=1411441 for reference.
vpl_run.sh
#!/bin/bash echo "#!/bin/bash" > vpl_execution if [ -f ./vpl_evaluate.sh ] ; then echo "python2.7 evaluateProgram.py" >> vpl_execution else echo "python2.7 runProgram.py" >> vpl_execution fi chmod +x vpl_execution
runProgram.py
# runProgram.py # students, do not modify this file. It will be used to test your program. import sys #--- run the program to test --- import Lab_9_Additional #--- display the contents of the sample.txt file which should #--- have been modified by the program under test. print "\n---Output.txt---" try: file = open( "Output.txt", "r" ) print file.read(), file.close() except IOError as e: print "I/O error({0}): {1}".format(e.errno, e.strerror) except: print "Unexpected error:", sys.exc_info()[0] raise
evaluateProgram.py
# evaluateProgram.py # students, do not modify this file. It will be used to test your program. import sys #--- capture the stdout of the program to test into a file saveStdOut = sys.stdout sys.stdout = open( "fakestdout.txt", "w" ) #--- run the program to test --- import Lab_9_Additional #--- get rid of whatever it printed on the stdout --- sys.stdout = saveStdOut #--- display the contents of the sample.txt file which should #--- have been modified by the program under test. try: file = open( "Output.txt", "r" ) print file.read(), file.close() except IOError as e: print "I/O error({0}): {1}".format(e.errno, e.strerror) except: print "Unexpected error:", sys.exc_info()[0] raise
Bash Quick Solutions
- To compare the student's output to the expected output, but not worry about order of lines output, or whitespace
# code included in vpl_evaluate.sh. All $-signs prefixed with back-slash. # $user contains output from student program # $expected contains expected output cat \$user | sort | xargs > \${user}_sorted cat \$expected | sort | xargs > \${expected}_sorted diff -y -w -B --ignore-all-space \${user}_sorted \${expected}_sorted > diff.out if ((\$? > 0)); then echo "Comment :=>> Your output is incorrect." echo "Comment :=>> ---------------" echo "Comment :=>>- Your output:" echo "Comment :=>> ---------------" echo "<|--" cat \$user echo "--|>" echo "" echo "Comment :=>> ---------------" echo "Comment :=>>- Expected output " echo "Comment :=>> ---------------" echo "<|--" cat \$expected echo "--|>" else echo "Comment :=>> Correct output." noTestsPassed=\$((noTestsPassed+1)) fi
- To remove comments from a Java program, including /* ... */, /** ... */, and //
prog1=yourprogramname # the name of the java code, '''without''' the ".java" extension cat $prog1.java | sed 's://.*$::g' | sed '/\/\*\*/,/\*\// {s/.*\*\/.*//p; d}' > _$prog1.java
- To remove the comments from an assembly program (.asm)
prog1=yourprogramname # then name of the asm code, '''without''' the ".asm" extension cat ${prog1}.asm | sed 's:;.*$::g' > _${prog1}.asm
- To stop programs with endless loops, use the timeout command.
- (Note: the \$ are required inside vpl_evaluate.sh when creating vpl_execute.sh)
timeout 30 java \$prog1 < dataIn.txt &> user.out retcode=\$? #--- if we killed the waiter, everything is good --- if [ \$retcode -eq 124 ]; then echo "Comment :=>>-Your program has stopped working." else echo "Comment :=>>-Your program ran to completion." fi
- To add @suppresswarnings() to Java programs that don't contain it
addSuppressWarnings() {
if [ "\$1" ]; then # if there's a parameter
prog=\$1
if grep SuppressWarnings $prog ; then
:
else
#echo "Not found"
sed -i 's/^class/@SuppressWarnings("unchecked")\nclass/' \${prog}
sed -i 's/^public class/@SuppressWarnings("unchecked")\npublic class/' \${prog}
fi
fi
}
###### (and in the code of ''vpl_evaluate.sh'', call function this way: ######
# --- add a @SuppressWarnings( "unchecked" ) label
addSuppressWarnings( \${prog1}.java )
Output of Evaluate Step
- The output of the Evaluate action is a bit illogical:
+------------------------------+ | 3 tests run/ 0 tests failed | +------------------------------+
- To change it into something more intuitive, such as this:
+------------------------------+ | 3 tests run/ 3 tests passed | +------------------------------+
- we simply need to modify the code of vpl_evaluate.cpp on the moodle server, in /var/www/html/moodle/mod/vpl/jail/default_scripts (or whatever directory the vpl files reside).
- The diff of the modification is shown below:
1058,1059c1058,1060 < printf(">| %2d %s run/%2d %s failed |\n", < nruns, nruns==1?stest[0]:stest[1], nerrors, nerrors==1?stest[0]:stest[1]); --- > printf(">| %2d %s run/%2d %s passed |\n", > nruns, nruns==1?stest[0]:stest[1], nruns-nerrors, > (nruns-nerrors)==1?stest[0]:stest[1]);
Generating a vpl_evaluate.cases File Quickly
- Recommendations
- write a solution program first.
- run it on several cases you want to test your students' programs with.
- capture the outputs of the solution program on different cases. Copy paste into the vpl_evaluate.cases file. Don't worry yet about making it exactly correct.
- evaluate the solution program. You will get mismatches between the generated output and the expected output. Just copy/paste the generated output back into the vpl_evaluate.cases file. You will also be able to include the extra characters generated by the input statements.
- You should be ready to evaluate the solution program and get 100/100 as a grade.
Creating a vpl_execution script that measures the size of a program
Imagine that the assignment for the student is to create a program that solves a particular problem (which can be tested separately for the correctness of its output), but the requirement is to make the program executable as small as possible.
The grade given to the submitted program will be 100 if the size is equal to the size of the solution program, and some lower grade depending on how large the file is.
Here's a bash script that will do the trick
#! /bin/bash # $file contains the name of the executable. The executable # is created by assembling ${file}.asm, generating ${file}.o, # and linking ${file}.o. # The script outputs 0 if the executable crashes, or just # an integer that is the size of the executable, in bytes, if # it runs correctly. file="printStars" # whatever executable is to be tested # assemble and link the executable nasm -f elf -F stabs ${file}.asm ld -melf_i386 -o $file ${file}.o # run the executable in a subshell to catch any segmentation fault error # that may result. { ./$file &> /dev/null ; } &> log # if an error occured, automatically return 0 if [ $? -eq 139 ]; then echo "0"; exit 0; fi # print out the size of the file size=$(stat -c%s "$file") echo $size
The part of the script that reads { ./$file &> /dev/null ; } &> log is a bit complex. The reason for this is that we want to catch a potential crash by the program being tested, prevent it from outputting some segmentation fault message that would have come up, and automatically output 0 for it, and not its size.
If the program runs correctly, its output is discarded and the script simply outputs the size in bytes of the executable.
Creating a vpl_execution Script that Outputs the Grade a Program Based on its Size
This time we take the previous version of vpl_execution and make it output the grade (0 to 100) based on how small the program executable is. The smallest possible size is assumed to be known.
#! /bin/bash # $file contains the name of the executable. The executable # is created by assembling ${file}.asm, generating ${file}.o, # and linking ${file}.o. # The script outputs 0 if the executable crashes, or just # an integer that is the size of the executable, in bytes, if # it runs correctly. file="printStars" # whatever executable is to be tested bestSize=1065 # the size of the executable version of the solution program # in bytes. # assemble and link the executable nasm -f elf -F stabs ${file}.asm ld -melf_i386 -o $file ${file}.o # run the executable in a subshell to catch any segmentation fault error # that may result. { ./$file &> /dev/null ; } &> log # if an error occured, automatically return 0 if [ $? -eq 139 ]; then echo "0"; exit 0; fi # print out the size of the file size=$(stat -c%s "$file") #echo $size # compare size to best size recorded and output # grade based on size. increment=20 # size increment that takes away 10 more points # from the grade # possible grades grades=( 100 90 80 70 60 50 40 30 20 10 0 ) # find bin corresponding to size of executable, based on # $bestSize, and bins of width $increment for i in {0..10} ; do bestSize=$(($bestSize+$increment)) if [ $size -le $bestSize ] ; then echo ${grades[$i]} exit 0 fi done echo "0"
Useful Bash Functions for vpl_evaluate.sh
Function to remove non numbers from a file
# function that removes non-digits, extra spaces, and extra blank lines from text file.
# parameter is the name of a text file (typically output of user program, or output of
# solution program).
cleanup () {
if [ "\$1" ]; then # if there's a parameter
#--- remove non numbers and non minus---
cat \$1 | sed 's/[^0-9*.0-9\ ]*//g' > dummy.out
cp dummy.out \$1
#--- remove multiple spaces ---
cat \$1 | sed 's/ */ /g' > dummy.out
cp dummy.out \$1
cat \$1 | sed 's/^[ \t]*//' > dummy.out
cp dummy.out \$1
#--- remove blank lines ---
cat \$1 | sed '/^\s*\$/d' > dummy.out
cp dummy.out \$1
fi
}
Report discrepancy between user and solution output
# function that prints the difference between user and expected output
# \$TEMP is a parameter used to test
incorrectOutput() {
echo "Comment :=>>- Your output is incorrect."
#--- display test file ---
echo "<|--"
echo "Your program tested with \$TEMP"
echo "--|>"
echo "Comment :=>> ---------------"
echo "Comment :=>>- Your output:"
echo "Comment :=>> ---------------"
echo "<|--"
cat userOut.org
echo "--|>"
echo ""
echo "Comment :=>> ---------------"
echo "Comment :=>>- Expected output: "
echo "Comment :=>> ---------------"
echo "<|--"
cat expectedOut.org
echo "--|>"
}
Loop and Test Program with 3 Different Parameters
for TEMP in \$TEMP1 \$TEMP2 \$TEMP3 ; do
# create input for the program
echo \$TEMP > input
# =================================================
# generate user output and expected output
\$python \$prog < input > userOut
\$python \$solutionProg < input > expectedOut
cp userOut userOut.org
cp expectedOut expectedOut.org
cleanup userOut
cleanup expectedOut
#--- compute difference ---
diff -y -w --ignore-all-space userOut expectedOut > diff.out
#--- reject if different ---
if ((\$? > 0)); then
incorrectOutput
grade=65
# --------------------- REWARD IF CORRECT OUTPUT -----------------
else
#--- good output ---
echo "Comment :=>>- Congrats, your output is correct."
echo "Comment :=>> --------------------------------."
echo "<|--"
echo "Your program tested with \$TEMP"
echo ""
cat userOut
echo "--|>"
grade=100
fi
done
Report Grade and Max it to 100 if Above
# Report grade
if (( grade > 100 )); then
grade=100
fi
echo "Grade :=>> \$grade"
Pick 3 Different Random Inputs Test Values
# =================================================
# Pick 3 random inputs to test program with
TEMP1=\$RANDOM
let "TEMP1 %= 30"
let "TEMP1 = TEMP1 + 6"
TEMP2=\$RANDOM
let "TEMP2 %= 30"
let "TEMP2 = 0 - TEMP2"
TEMP3=\$RANDOM
let "TEMP3 %= 30"
let "TEMP3 = TEMP3 + 2"
TEMP3=\${TEMP3}.5