Difference between revisions of "Tutorial Moodle VPL Tips & Tricks"

From dftwiki3
Jump to: navigation, search
(→‎Bash Quick Solutions)
(→‎How to Differentiate Between Run and Evaluate Modes?)
 
(6 intermediate revisions by the same user not shown)
Line 20: Line 20:
 
<br />
 
<br />
 
=Tips & Tricks=
 
=Tips & Tricks=
 +
 +
==How to Differentiate Between Run and Evaluate Modes?==
 +
<br />
 +
This solution was provided to me by Sashini Herath, of CS Department at the University of Moratuwa, in Sri Lanka (http://cse.mrt.ac.lk), with insights from Juan Carlos Rodríguez-del-Pino.  Please see [https://moodle.org/mod/forum/discuss.php?d=154988&parent=1411441 https://moodle.org/mod/forum/discuss.php?d=154988&parent=1411441] for reference.
 +
<br />
 +
===vpl_run.sh===
 +
<br />
 +
::<source lang="bash">
 +
#!/bin/bash
 +
echo "#!/bin/bash" > vpl_execution
 +
if [ -f ./vpl_evaluate.sh ] ; then
 +
      echo "python2.7 evaluateProgram.py" >> vpl_execution
 +
else
 +
      echo "python2.7 runProgram.py" >> vpl_execution
 +
fi
 +
chmod +x vpl_execution
 +
</source>
 +
<br />
 +
===runProgram.py===
 +
<br />
 +
::<source lang="python">
 +
# runProgram.py
 +
# students, do not modify this file.  It will be used to test your program.
 +
 +
import sys
 +
 +
#--- run the program to test ---
 +
import Lab_9_Additional
 +
 +
#--- display the contents of the sample.txt file which should
 +
#--- have been modified by the program under test.
 +
 +
print "\n---Output.txt---"
 +
try:
 +
    file = open( "Output.txt", "r" )
 +
    print file.read(),
 +
    file.close()
 +
except IOError as e:
 +
    print "I/O error({0}): {1}".format(e.errno, e.strerror)
 +
except:
 +
    print "Unexpected error:", sys.exc_info()[0]
 +
    raise
 +
</source>
 +
<br />
 +
===evaluateProgram.py===
 +
<br />
 +
::<source lang="python">
 +
# evaluateProgram.py
 +
# students, do not modify this file.  It will be used to test your program.
 +
 +
import sys
 +
 +
#--- capture the stdout of the program to test into a file
 +
saveStdOut = sys.stdout
 +
sys.stdout = open( "fakestdout.txt", "w" )
 +
 +
#--- run the program to test ---
 +
import Lab_9_Additional
 +
 +
#--- get rid of whatever it printed on the stdout ---
 +
sys.stdout = saveStdOut
 +
 +
#--- display the contents of the sample.txt file which should
 +
#--- have been modified by the program under test.
 +
 +
try:
 +
    file = open( "Output.txt", "r" )
 +
    print file.read(),
 +
    file.close()
 +
except IOError as e:
 +
    print "I/O error({0}): {1}".format(e.errno, e.strerror)
 +
except:
 +
    print "Unexpected error:", sys.exc_info()[0]
 +
    raise
 +
</source>
 +
<br />
  
 
==Bash Quick Solutions==
 
==Bash Quick Solutions==
 +
<br />
 +
; To compare the student's output to the expected output, but not worry about order of lines output, or whitespace:
 +
<br />
 +
::<source lang="bash">
 +
  # code included in vpl_evaluate.sh.  All $-signs prefixed with back-slash.
 +
  # $user contains output from student program
 +
  # $expected contains expected output
 +
  cat \$user | sort | xargs  > \${user}_sorted
 +
  cat \$expected | sort | xargs > \${expected}_sorted
 +
  diff -y -w -B --ignore-all-space \${user}_sorted \${expected}_sorted > diff.out
 +
 +
  if ((\$? > 0)); then
 +
      echo "Comment :=>> Your output is incorrect."
 +
      echo "Comment :=>> ---------------"
 +
      echo "Comment :=>>- Your output:"
 +
      echo "Comment :=>> ---------------"
 +
      echo "<|--"
 +
      cat \$user
 +
      echo "--|>"
 +
      echo ""
 +
      echo "Comment :=>> ---------------"
 +
      echo "Comment :=>>- Expected output "
 +
      echo "Comment :=>> ---------------"
 +
      echo "<|--"
 +
      cat \$expected
 +
      echo "--|>"
 +
  else
 +
      echo "Comment :=>> Correct output."
 +
      noTestsPassed=\$((noTestsPassed+1))
 +
  fi
 +
</source>
 
<br />
 
<br />
 
; To remove  comments from a '''Java''' program, including /* ... */, /** ... */, and //:
 
; To remove  comments from a '''Java''' program, including /* ... */, /** ... */, and //:
Line 40: Line 147:
 
; To stop programs with endless loops, use the '''timeout''' command.
 
; To stop programs with endless loops, use the '''timeout''' command.
 
<br />
 
<br />
 +
:(Note: the \$ are required inside vpl_evaluate.sh when creating vpl_execute.sh)
 
::<source lang="bash">
 
::<source lang="bash">
−
   timeout 30 java \${prog1} < data\${i}.txt  &> user.out
+
   timeout 30 java \$prog1 < dataIn.txt  &> user.out
 
   retcode=\$?
 
   retcode=\$?
  
Line 47: Line 155:
 
   if [ \$retcode -eq  124 ]; then
 
   if [ \$retcode -eq  124 ]; then
 
       echo "Comment :=>>-Your program has stopped working."
 
       echo "Comment :=>>-Your program has stopped working."
−
      timeoutOutput \$i
 
−
      continue
 
 
   else
 
   else
 
       echo "Comment :=>>-Your program ran to completion."
 
       echo "Comment :=>>-Your program ran to completion."
Line 209: Line 315:
 
echo "0"
 
echo "0"
 
</source>
 
</source>
 +
<!-- ---------------------------------------------------------------------------------------------------------------- -->
 
<br />
 
<br />
 +
 +
=Useful Bash Functions for vpl_evaluate.sh=
 +
<br />
 +
==Function to remove non numbers from a file ==
 +
<br />
 +
<source lang="bash">
 +
# function that removes non-digits, extra spaces, and extra blank lines from text file.
 +
# parameter is the name of a text file (typically output of user program, or output of
 +
# solution program).
 +
cleanup () {
 +
    if [ "\$1" ]; then  # if there's a parameter
 +
       
 +
        #--- remove non numbers and non minus---
 +
        cat \$1 | sed 's/[^0-9*.0-9\ ]*//g' > dummy.out
 +
        cp dummy.out \$1
 +
 +
        #--- remove multiple spaces ---
 +
        cat \$1 | sed 's/  */ /g' > dummy.out
 +
        cp dummy.out \$1
 +
 +
        cat \$1 | sed 's/^[ \t]*//' > dummy.out
 +
        cp dummy.out \$1
 +
 +
        #--- remove blank lines ---
 +
        cat \$1 | sed '/^\s*\$/d' > dummy.out
 +
        cp dummy.out \$1
 +
  fi
 +
}
 +
 +
</source>
 +
<br />
 +
<br />
 +
== Report discrepancy between user and solution output ==
 +
<br />
 +
<source lang="bash">
 +
# function that prints the difference between user and expected output
 +
# \$TEMP is a parameter used to test
 +
incorrectOutput() {
 +
        echo "Comment :=>>- Your output is incorrect."
 +
        #--- display test file ---
 +
        echo "<|--"
 +
        echo "Your program tested with \$TEMP"
 +
        echo "--|>"
 +
 +
        echo "Comment :=>> ---------------"
 +
        echo "Comment :=>>- Your output:"
 +
        echo "Comment :=>> ---------------"
 +
        echo "<|--"
 +
        cat userOut.org
 +
        echo "--|>"
 +
        echo ""
 +
        echo "Comment :=>> ---------------"
 +
        echo "Comment :=>>- Expected output: "
 +
        echo "Comment :=>> ---------------"
 +
        echo "<|--"
 +
        cat expectedOut.org
 +
        echo "--|>"
 +
 +
}
 +
 +
</source>
 +
<br />
 +
<br />
 +
== Loop and Test Program with 3 Different Parameters ==
 +
<br />
 +
<source lang="bash">
 +
for TEMP in \$TEMP1 \$TEMP2 \$TEMP3 ; do
 +
  # create input for the program
 +
  echo \$TEMP > input
 +
 +
  # =================================================
 +
  # generate user output and expected output
 +
  \$python \$prog < input > userOut
 +
  \$python \$solutionProg < input > expectedOut
 +
 +
  cp userOut userOut.org
 +
  cp expectedOut expectedOut.org
 +
 +
  cleanup userOut
 +
  cleanup expectedOut
 +
 +
  #--- compute difference ---
 +
  diff -y -w --ignore-all-space userOut expectedOut > diff.out
 +
 +
  #--- reject if different ---
 +
  if ((\$? > 0)); then
 +
      incorrectOutput
 +
      grade=65
 +
      # --------------------- REWARD IF CORRECT OUTPUT -----------------
 +
  else
 +
      #--- good output ---
 +
      echo "Comment :=>>- Congrats, your output is correct."
 +
      echo "Comment :=>> --------------------------------."
 +
      echo "<|--"                                                                           
 +
      echo "Your program tested with \$TEMP"
 +
      echo ""
 +
      cat userOut
 +
      echo "--|>"
 +
      grade=100
 +
  fi
 +
 +
done
 +
</source>
 +
<br />
 +
<br />
 +
== Report Grade and Max it to 100 if Above ==
 +
<br />
 +
<source lang="bash">
 +
# Report grade
 +
if (( grade > 100 )); then
 +
  grade=100
 +
fi
 +
echo "Grade :=>> \$grade"
 +
 +
 +
</source>
 +
<br />
 +
<br />
 +
== Pick 3 Different Random Inputs Test Values ==
 +
<br />
 +
<source lang="bash">
 +
# =================================================
 +
# Pick 3 random inputs to test program with
 +
TEMP1=\$RANDOM
 +
let "TEMP1 %= 30"
 +
let "TEMP1 = TEMP1 + 6"
 +
TEMP2=\$RANDOM
 +
let "TEMP2 %= 30"
 +
let "TEMP2 = 0 - TEMP2"
 +
TEMP3=\$RANDOM
 +
let "TEMP3 %= 30"
 +
let "TEMP3 = TEMP3 + 2"
 +
TEMP3=\${TEMP3}.5
 +
 +
</source>
 +
<br />
 +
 
<br />
 
<br />
 
<br />
 
<br />

Latest revision as of 11:00, 30 March 2017

--D. Thiebaut (talk) 20:31, 11 June 2014 (EDT)



MoodleVPLLogo.png


Moodle VPL Tutorials



This is a collection of tips, tricks, observations, recommendations, and modifications related to using VPL with Moodle.


Tips & Tricks

How to Differentiate Between Run and Evaluate Modes?


This solution was provided to me by Sashini Herath, of CS Department at the University of Moratuwa, in Sri Lanka (http://cse.mrt.ac.lk), with insights from Juan Carlos Rodríguez-del-Pino. Please see https://moodle.org/mod/forum/discuss.php?d=154988&parent=1411441 for reference.

vpl_run.sh


#!/bin/bash
echo "#!/bin/bash" > vpl_execution
if [ -f ./vpl_evaluate.sh ] ; then
      echo "python2.7 evaluateProgram.py" >> vpl_execution
else
      echo "python2.7 runProgram.py" >> vpl_execution
fi
chmod +x vpl_execution


runProgram.py


# runProgram.py
# students, do not modify this file.  It will be used to test your program.

import sys

#--- run the program to test ---
import Lab_9_Additional

#--- display the contents of the sample.txt file which should 
#--- have been modified by the program under test.

print "\n---Output.txt---"
try:
    file = open( "Output.txt", "r" )
    print file.read(), 
    file.close()
except IOError as e:
    print "I/O error({0}): {1}".format(e.errno, e.strerror)
except:
    print "Unexpected error:", sys.exc_info()[0]
    raise


evaluateProgram.py


# evaluateProgram.py
# students, do not modify this file.  It will be used to test your program.

import sys

#--- capture the stdout of the program to test into a file
saveStdOut = sys.stdout
sys.stdout = open( "fakestdout.txt", "w" )

#--- run the program to test ---
import Lab_9_Additional

#--- get rid of whatever it printed on the stdout ---
sys.stdout = saveStdOut

#--- display the contents of the sample.txt file which should 
#--- have been modified by the program under test.

try:
    file = open( "Output.txt", "r" )
    print file.read(), 
    file.close()
except IOError as e:
    print "I/O error({0}): {1}".format(e.errno, e.strerror)
except:
    print "Unexpected error:", sys.exc_info()[0]
    raise


Bash Quick Solutions


To compare the student's output to the expected output, but not worry about order of lines output, or whitespace


   # code included in vpl_evaluate.sh.  All $-signs prefixed with back-slash.
   # $user contains output from student program
   # $expected contains expected output
   cat \$user | sort | xargs  > \${user}_sorted
   cat \$expected | sort | xargs > \${expected}_sorted
   diff -y -w -B --ignore-all-space \${user}_sorted \${expected}_sorted > diff.out

   if ((\$? > 0)); then
      echo "Comment :=>> Your output is incorrect."
      echo "Comment :=>> ---------------"
      echo "Comment :=>>- Your output:"
      echo "Comment :=>> ---------------"
      echo "<|--"
      cat \$user
      echo "--|>"
      echo ""
      echo "Comment :=>> ---------------"
      echo "Comment :=>>- Expected output "
      echo "Comment :=>> ---------------"
      echo "<|--"
      cat \$expected
      echo "--|>"
   else
      echo "Comment :=>> Correct output."
      noTestsPassed=\$((noTestsPassed+1))
   fi


To remove comments from a Java program, including /* ... */, /** ... */, and //


  
  prog1=yourprogramname         # the name of the java code, '''without''' the ".java" extension
  cat $prog1.java | sed 's://.*$::g' | sed '/\/\*\*/,/\*\// {s/.*\*\/.*//p; d}' > _$prog1.java


To remove the comments from an assembly program (.asm)


 prog1=yourprogramname         # then name of the asm code, '''without''' the ".asm" extension
 cat ${prog1}.asm | sed 's:;.*$::g'  > _${prog1}.asm


To stop programs with endless loops, use the timeout command.


(Note: the \$ are required inside vpl_evaluate.sh when creating vpl_execute.sh) 
  timeout 30 java \$prog1 < dataIn.txt  &> user.out
  retcode=\$?

  #--- if we killed the waiter, everything is good ---
  if [ \$retcode -eq  124 ]; then
      echo "Comment :=>>-Your program has stopped working."
  else
      echo "Comment :=>>-Your program ran to completion."
  fi


To add @suppresswarnings() to Java programs that don't contain it


addSuppressWarnings() {
    if [ "\$1" ]; then   # if there's a parameter
        prog=\$1
        if grep SuppressWarnings $prog ; then
           :
        else
           #echo "Not found"
           sed -i 's/^class/@SuppressWarnings("unchecked")\nclass/'  \${prog}
           sed -i 's/^public class/@SuppressWarnings("unchecked")\npublic class/'  \${prog}
        fi
    fi
}

###### (and in the code of ''vpl_evaluate.sh'', call function this way: ######
# --- add a @SuppressWarnings( "unchecked" ) label
addSuppressWarnings( \${prog1}.java )


Output of Evaluate Step

  • The output of the Evaluate action is a bit illogical:
 +------------------------------+
 |  3 tests run/ 0 tests failed   | 
 +------------------------------+
  • To change it into something more intuitive, such as this:
 +------------------------------+
 |  3 tests run/ 3 tests passed  |  
 +------------------------------+
we simply need to modify the code of vpl_evaluate.cpp on the moodle server, in /var/www/html/moodle/mod/vpl/jail/default_scripts (or whatever directory the vpl files reside).
  • The diff of the modification is shown below:
1058,1059c1058,1060
< 				printf(">| %2d %s run/%2d %s failed |\n",
< 						nruns, nruns==1?stest[0]:stest[1], nerrors, nerrors==1?stest[0]:stest[1]);
---
> 				printf(">| %2d %s run/%2d %s passed |\n",
> 						nruns, nruns==1?stest[0]:stest[1], nruns-nerrors, 
> 				                (nruns-nerrors)==1?stest[0]:stest[1]);


Generating a vpl_evaluate.cases File Quickly


Recommendations
  1. write a solution program first.
  2. run it on several cases you want to test your students' programs with.
  3. capture the outputs of the solution program on different cases. Copy paste into the vpl_evaluate.cases file. Don't worry yet about making it exactly correct.
  4. evaluate the solution program. You will get mismatches between the generated output and the expected output. Just copy/paste the generated output back into the vpl_evaluate.cases file. You will also be able to include the extra characters generated by the input statements.
  5. You should be ready to evaluate the solution program and get 100/100 as a grade.


Creating a vpl_execution script that measures the size of a program


Imagine that the assignment for the student is to create a program that solves a particular problem (which can be tested separately for the correctness of its output), but the requirement is to make the program executable as small as possible. The grade given to the submitted program will be 100 if the size is equal to the size of the solution program, and some lower grade depending on how large the file is.
Here's a bash script that will do the trick 

#! /bin/bash
# $file contains the name of the executable.  The executable
# is created by assembling ${file}.asm, generating ${file}.o,
# and linking ${file}.o.
# The script outputs 0 if the executable crashes, or just
# an integer that is the size of the executable, in bytes, if
# it runs correctly.

file="printStars"  # whatever executable is to be tested

# assemble and link the executable
nasm -f elf -F stabs ${file}.asm
ld -melf_i386 -o $file ${file}.o

# run the executable in a subshell to catch any segmentation fault error
# that may result.
{ ./$file &> /dev/null ; } &> log

# if an error occured, automatically return 0
if [ $? -eq 139 ]; then
    echo "0";
    exit 0;
fi

# print out the size of the file
size=$(stat -c%s "$file")
echo $size


The part of the script that reads { ./$file &> /dev/null ; } &> log is a bit complex. The reason for this is that we want to catch a potential crash by the program being tested, prevent it from outputting some segmentation fault message that would have come up, and automatically output 0 for it, and not its size.

If the program runs correctly, its output is discarded and the script simply outputs the size in bytes of the executable.

Creating a vpl_execution Script that Outputs the Grade a Program Based on its Size


This time we take the previous version of vpl_execution and make it output the grade (0 to 100) based on how small the program executable is. The smallest possible size is assumed to be known. 

#! /bin/bash
# $file contains the name of the executable.  The executable
# is created by assembling ${file}.asm, generating ${file}.o,
# and linking ${file}.o.
# The script outputs 0 if the executable crashes, or just
# an integer that is the size of the executable, in bytes, if
# it runs correctly.

file="printStars"  # whatever executable is to be tested
bestSize=1065    # the size of the executable version of the solution program
                         # in bytes.

# assemble and link the executable
nasm -f elf -F stabs ${file}.asm
ld -melf_i386 -o $file ${file}.o

# run the executable in a subshell to catch any segmentation fault error
# that may result.
{ ./$file &> /dev/null ; } &> log

# if an error occured, automatically return 0
if [ $? -eq 139 ]; then
    echo "0";
    exit 0;
fi

# print out the size of the file
size=$(stat -c%s "$file")
#echo $size

# compare size to best size recorded and output
# grade based on size.

increment=20   # size increment that takes away 10 more points
               # from the grade

# possible grades
grades=( 100 90 80 70 60 50 40 30 20 10 0 )

# find bin corresponding to size of executable, based on
# $bestSize, and bins of width $increment
for i in {0..10} ; do
    bestSize=$(($bestSize+$increment))
    if [ $size -le $bestSize ] ; then
	echo ${grades[$i]}
	exit 0
    fi  
done
echo "0"


Useful Bash Functions for vpl_evaluate.sh


Function to remove non numbers from a file


# function that removes non-digits, extra spaces, and extra blank lines from text file.
# parameter is the name of a text file (typically output of user program, or output of
# solution program).
cleanup () {
    if [ "\$1" ]; then   # if there's a parameter
        
        #--- remove non numbers and non minus---
        cat \$1 | sed 's/[^0-9*.0-9\ ]*//g' > dummy.out
        cp dummy.out \$1
 
        #--- remove multiple spaces --- 
        cat \$1 | sed 's/  */ /g' > dummy.out
        cp dummy.out \$1
 
        cat \$1 | sed 's/^[ \t]*//' > dummy.out
        cp dummy.out \$1

        #--- remove blank lines ---
        cat \$1 | sed '/^\s*\$/d' > dummy.out
        cp dummy.out \$1
   fi
}



Report discrepancy between user and solution output


# function that prints the difference between user and expected output
# \$TEMP is a parameter used to test
incorrectOutput() {
        echo "Comment :=>>- Your output is incorrect."
        #--- display test file ---
        echo "<|--" 
        echo "Your program tested with \$TEMP"
        echo "--|>"

        echo "Comment :=>> ---------------"
        echo "Comment :=>>- Your output:"
        echo "Comment :=>> ---------------"
        echo "<|--"
        cat userOut.org
        echo "--|>"
        echo ""
        echo "Comment :=>> ---------------"
        echo "Comment :=>>- Expected output: "
        echo "Comment :=>> ---------------"
        echo "<|--"
        cat expectedOut.org
        echo "--|>"

}



Loop and Test Program with 3 Different Parameters


for TEMP in \$TEMP1 \$TEMP2 \$TEMP3 ; do
   # create input for the program
   echo \$TEMP > input

   # =================================================
   # generate user output and expected output
   \$python \$prog < input > userOut
   \$python \$solutionProg < input > expectedOut

   cp userOut userOut.org
   cp expectedOut expectedOut.org

   cleanup userOut
   cleanup expectedOut

   #--- compute difference --- 
   diff -y -w --ignore-all-space userOut expectedOut > diff.out

   #--- reject if different ---
   if ((\$? > 0)); then
      incorrectOutput 
      grade=65
      # --------------------- REWARD IF CORRECT OUTPUT -----------------
   else
      #--- good output ---
      echo "Comment :=>>- Congrats, your output is correct."
      echo "Comment :=>> --------------------------------."
      echo "<|--"                                                                            
      echo "Your program tested with \$TEMP"
      echo ""
      cat userOut
      echo "--|>"
      grade=100
   fi

done



Report Grade and Max it to 100 if Above


# Report grade
if (( grade > 100 )); then
   grade=100
fi
echo "Grade :=>> \$grade"



Pick 3 Different Random Inputs Test Values


# =================================================
# Pick 3 random inputs to test program with
TEMP1=\$RANDOM
let "TEMP1 %= 30"
let "TEMP1 = TEMP1 + 6"
TEMP2=\$RANDOM
let "TEMP2 %= 30"
let "TEMP2 = 0 - TEMP2"
TEMP3=\$RANDOM
let "TEMP3 %= 30"
let "TEMP3 = TEMP3 + 2"
TEMP3=\${TEMP3}.5








Retrieved from "http://dominiquethiebaut.com/dftwiki3/index.php?title=Tutorial_Moodle_VPL_Tips_%26_Tricks&oldid=30518"