Tutorial: Moodle VPL -- Java Tester Class

--D. Thiebaut (talk) 12:58, 13 September 2014 (EDT)

---

---

Moodle VPL Tutorials

---

This tutorial generates a VPL activity that allows automatic evaluation of a Java program with 1 class submitted by the student, which is derived from another class (given to the students, but not submitted by the students). The program is tested by a class provided by the instructor. This class outputs a series of lines. The vpl_run.sh script tests how many keywords appear in the output and decides on the correctness of the program from the number of keywords output, and not on the exact output by the student program.

Setup

• Moodle Version 2.7 + (Build: 20140529)
• VPL Version 3.1
• For details on how Moodle and VPL were installed, go to this page.

Settings

• Homework 1, Problem 1
• Due date: Thursday, September 18, 2014, 1:00 PM
• Requested files: Hw1_1.java (Download)
• Type of work: Individual work
• Grade settings: Maximum grade: 100
• Run: Yes Evaluate: Yes

Requested files

Hw1_1.java

    1 // paste your code below...
    2

Execution files

vpl_run.sh

#! /bin/bash
cat > vpl_execution << 'EOF'
#! /bin/bash
javac -J-Xmx128m VPLFakeTester.java 
javac -J-Xmx128m VPLJavaTester.java 
java VPLJavaTester Hw1_1.java patterns.txt
      
EOF
       
chmod +x vpl_execution

vpl_evaluate.sh

#! /bin/bash
cat > vpl_execution << 'EOF'
#! /bin/bash
javac -J-Xmx128m VPLFakeTester.java 
javac -J-Xmx128m VPLJavaTester.java 
java VPLJavaTester Hw1_1.java patterns.txt
     
EOF
    
chmod +x vpl_execution

VPLJavaTester.java

import java.io.ByteArrayOutputStream;
import java.io.PrintStream;

/**
 * Test a Java VPL program
 * @author thiebaut
 * 
 */
public class VPLJavaTester extends Hw1_1 { 						// <=== extends the student class

	static final String expected = "1 7 21 35 35 21 7 1";	// <=== perfect output
	
	/**
	 * formats strings to create VPL comments
	 * @param s the comment for the student
	 */
	private static void comment(String  s ) {
	    String[] lines = s.split( "\n" );
        System.out.println(  s );
	    //for ( int i=0; i<lines.length; i++ ) {
	        //System.out.println( "Comment:=>>" + lines[i] );
	    //}
	    //System.out.println( "<|-" );
	    //System.out.println( s );
	    //System.out.println( "-|>" );
	}

	/**
	 * formats a number to create a VPL grade
	 * @param num
	 */
	private static void grade( int num ) {
	    System.out.println( "Grade :=>> " + num );
	}

	/**
	 * takes the output of the student program, tests it against the expected String
	 * and generates a grade, along with a comment.
	 * @param studentOutput
	 */
	private static void testOutputAndGrade( String studentOutput ) {
	
		//--- clean up the student programs output ---
		String cleanStudentOutput = studentOutput.trim().replaceAll("\\s+", " ");
		
		int expectedCount = expected.split( "\\s+" ).length;
		
		//--- feedback ---
		comment( "Your program output:\n" + studentOutput );

		//--- perfect match ---
		if ( expected.equals( cleanStudentOutput ) ) {
			comment( "Correct output, congrats! Grade = 100/100" );
			grade( 100 );
			return;
		}
		
		//--- otherwise figure out something ---
		String[] indiv = cleanStudentOutput.split( " " );
		int count = 0;
		for ( int i=0; i<indiv.length ; i++ ) {
			if ( expected.indexOf( indiv[i] ) != -1 ) 
				 count++;
			//System.out.println( "indiv/count: " + indiv[i] + " " + count );
		}
		
		comment( "Your output contains roughly " + count + " out of " + expectedCount + " good numbers." );
		grade( (int) (count*100.0 / expectedCount) );
			
	}
	
	public static void main(String[] args) {

		if ( args.length == 0 )
			return;

		boolean isGenuine = VPLFakeTester.testIfGenuine( args );
		
		if ( !isGenuine ) {
			comment( "Your program does not compute the output the expected way." );
			grade( 0 );
			return;
		}

		//--- instantiate student submitted class ---
		Hw1_1 studentProgram = new Hw1_1();
		
		//--- capture stdout and stderr to string ---
		PrintStream orgOutStream   = System.out;
		PrintStream orgErrStream   = System.err;
		
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		PrintStream outputStream = new PrintStream(baos);
		System.setOut( outputStream );
		System.setErr( outputStream );
		
		// ======================================================= //
		//                  run student program                    //
		
		studentProgram.main( args );
		
		// ======================================================= //
		
		//--- get std-output and std-error back ---
		String output = baos.toString();
	    System.setOut( orgOutStream );
	    System.setErr( orgErrStream );
	    
	    //--- test the output against what's expected and grade program ---
		testOutputAndGrade( output );
	}

}

patterns.txt

This text file contains string patterns that might indicate possible cheating, or printing a string showing a result that should be computed instead.

1, 7, 21, 21
"7"
"21"

VPLFakeTester.java

/** 
 * VPLTester.java
 * D. Thiebaut
 * 
 * This program can be used to test a Java program with Moodle's VPL module.
 * It needs to be given the names of 2 files on the command line:
 * 1) the name of the file submitted by the student
 * 2) the name of a text file that will contain a series of patterns to test.
 * 
 * If the student program contains any of the patterns found in the pattern file
 * (1 pattern per line), then the program outputs the word "fake" and returns
 * 1 to the OS.  Otherwise it outputs "genuine" and returns 0. 
 * 
 */


import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Scanner;

 
/**
 * VPLTest class
 * Everything runs as static methods from main().
 * @author thiebaut
 *
 */
public class VPLFakeTester {

	//--- return codes ---
	final static int OK = 1;
	final static int FAKE = 2;

	
	/**
	 * removes java comments from the string code.
	 * @param code the string contains all the java code to process.  
	 * @return the string without the comments
 	 * @author: Andrew Vitkus, 
 	 * URL: http://stackoverflow.com/questions/23333031/removing-all-types-of-comments-in-java-file
	 */
	private static String  removeComments(String code) {
	    StringBuilder newCode = new StringBuilder();
	    try (StringReader sr = new StringReader(code)) {
	        boolean inBlockComment = false;
	        boolean inLineComment = false;
	        boolean out = true;

	        int prev = sr.read();
	        int cur;
	        for(cur = sr.read(); cur != -1; cur = sr.read()) {
	            if(inBlockComment) {
	                if (prev == '*' && cur == '/') {
	                    inBlockComment = false;
	                    out = false;
	                }
	            } else if (inLineComment) {
	                if (cur == '\r') { // start untested block
	                    sr.mark(1);
	                    int next = sr.read();
	                    if (next != '\n') {
	                        sr.reset();
	                    }
	                    inLineComment = false;
	                    out = false; // end untested block
	                } else if (cur == '\n') {
	                    inLineComment = false;
	                    out = false;
	                }
	            } else {
	                if (prev == '/' && cur == '*') {
	                    sr.mark(1); // start untested block
	                    int next = sr.read();
	                    //if (next != '*') {
	                        inBlockComment = true; // tested line (without rest of block)
	                    //}
	                    sr.reset(); // end untested block
	                } else if (prev == '/' && cur == '/') {
	                    inLineComment = true;
	                } else if (out){
	                    newCode.append((char)prev);
	                } else {
	                    out = true;
	                }
	            }
	            prev = cur;
	        }
	        if (prev != -1 && out && !inLineComment) {
	            newCode.append((char)prev);
	        }
	    } catch (IOException e) {
	        e.printStackTrace();
	    }

	    return newCode.toString();
	}
	
	/**
	 * Reads a text file from the directory dir, in the subdirectory subDir,
	 * and with name fileName.  The whole contents of the file is returned as a string
	 * @param dir the path where to find the file (must end with /)
	 * @param subDir the subdirectory (must end with /)
	 * @param fileName the file name
	 * @return the contents of the file as a string.  \n characters separate the lines.
	 */
	private static String readTextFile( String dir, String subDir, String fileName ) {
		
		//--- the contents of the file ---
		String javaText = "";
		
		
		try {
			Scanner in = new Scanner(new FileReader( dir + subDir + fileName ));
			while (in.hasNextLine()) {
	            javaText += in.nextLine() + "\n";
	        }
	        in.close();
		} catch (FileNotFoundException e) {
			//e.printStackTrace();
			//System.out.println( "Could not find file: " + subDir + fileName );
			return "";
		}
        //System.out.println( "Found file at " + dir + subDir + fileName );
		return javaText;
	}
	
	public static int testGenuine(String[] args)  {

		//--- variables ---
		String[] patterns = null;
		String fileName    = args[0];
		String patternFile = args[1];
		String currentDir = System.getProperty("user.dir") + "/";
		
		//--- read the java file, and remove its comments ---
		String javaText = readTextFile( currentDir, "", fileName );
		//if ( javaText.equals( "" ) ) {
		//	javaText = readTextFile( currentDir, "", fileName );
		//}
		
		
		if ( javaText.length() <= 1 )
			return OK;
		
		// System.out.println( javaText );
		javaText = removeComments( javaText );
		
		//--- read the pattern file ---
		String patrn = readTextFile( currentDir, "", patternFile );
		//if ( patrn.equals( "" ) ) {
		//	patrn = readTextFile( currentDir, "", patternFile );
		//}
		if ( patrn.length() <= 1 ) 
			return OK;
		
		patterns = patrn.split( "\n" );
		boolean genuine = true;
		for ( int i = 0; i< patterns.length ; i++) {
			//System.out.println( i + " " + patterns[i] );
			patrn = patterns[i];
			if ( javaText.indexOf( patrn ) != -1 ) {
				genuine = false;
				break;
			}
		}
		if ( genuine ) 
			return OK;
		
		// alas, a fake...
		return FAKE;
	}

	public static boolean testIfGenuine( String[] args ) {
		
		if ( args.length <= 0 ) {
			System.out.println( "no java file provided." );
			return true;
		}

		//--- get the student's java file name
		String fileName    = args[0];

		//--- check to see if the program fakes the computation ---
		int retCode = testGenuine( args );
		if ( retCode == OK )
			return true;
		return false;
	}
	
}