Add support for superscoring.

README.md

@@ -95,5 +95,10 @@ repo has changed since the last run. To forcibly regrade:
 
     rake grade cs123 mp1 regrade=yes
 
+You can enable superscoring as well, which will re-run tests that a student
+failed in an attempt to see if they can get a higher grade:
+
+    rake grade cs123 mp1 superscore=yes
+
 You can combine these arguments in any meaningful way. `semester` and
 `students` are also available options for `commit`.

lib/kgrader/assignment.rb

@@ -30,8 +30,13 @@ module KGrader
 
     def tests
       @tests ||= @config['grade'].map do |it|
-        script = File.join @root, it.keys.first + ".rb"
-        { :name => it.keys.first, :script => script, :max => it.values.first }
+        name = it.keys.first
+        opts = it.values.first
+        script = File.join @root, name + '.rb'
+        depends = *opts['depends'] || []
+        depfail = opts['depfail'] || "depends on #{name}"
+        { :name => name, :script => script, :max => opts['points'],
+          :depends => depends, :depfail => depfail }
       end
     end
 
@@ -47,7 +52,8 @@ module KGrader
     end
 
     def extra_comments
-      @config['commit']['comments'] || []
+      conf = @config['commit']
+      return *conf['comments'] || conf['comment'] || []
     end
 
     private

lib/kgrader/filesystem.rb

@@ -80,7 +80,7 @@ module KGrader
         File.write fn, "ungraded" if File.read(fn) == "graded"
       end
       FileUtils.rm_f Dir[File.join desk_dir, '*', '*', '*', '*', 'pending']
-      FileUtils.rm_f Dir[File.join desk_dir, '*', '*', '*', '*', '*.log']
+      FileUtils.rm_f Dir[File.join desk_dir, '*', '*', '*', '*', '*.log*']
     end
 
     # -------------------------------------------------------------------------

lib/kgrader/jail.rb

@@ -39,10 +39,10 @@ module KGrader
       [grade_wr, cmt_wr].each &:close
       Process.waitpid pid, 0
 
-      cmt_rd.read.split("\n").each { |cmt| yield cmt } if block_given?
       grade = grade_rd.read.strip.to_i
+      comments = cmt_rd.read.split("\n")
       [grade_rd, cmt_rd].each &:close
-      grade
+      return grade, comments
     end
 
     private

lib/kgrader/submission.rb

@@ -48,8 +48,8 @@ module KGrader
       nil
     end
 
-    def grade
-      grade_prep
+    def grade(superscore = false)
+      grade_prep superscore
       stage
       build
       test
@@ -61,13 +61,15 @@ module KGrader
     def commit
       if status == :graded && File.exists?(pendingfile)
         message = @assignment.commit_message @student
-        FileUtils.cp gradefile, File.join(repo, @assignment.report)
+        FileUtils.cp gradereport, File.join(repo, @assignment.report)
         @course.backend.commit repo, message, @assignment.report
         FileUtils.rm pendingfile
       end
       nil
     end
 
+    # -------------------------------------------------------------------------
+
     private
     def repo
       File.join @root, 'repo'
@@ -78,6 +80,10 @@ module KGrader
     end
 
     def gradefile
+      File.join @root, 'grade.json'
+    end
+
+    def gradereport
       File.join @root, 'grade.txt'
     end
 
@@ -109,19 +115,32 @@ module KGrader
       rev
     end
 
-    def grade_prep
-      @failure = false
-      @comments = []
+    # -------------------------------------------------------------------------
+
+    def grade_prep(superscore)
+      @done = false
+      @failed = false
+      @changed = !superscore || self.status == :ungraded
       @summary = nil
-      @tests = @assignment.tests.clone.each { |test| test[:score] = 0 }
+      @tests = @assignment.tests.clone.each do |test|
+        test[:score] = 0
+        test[:comments] = []
+      end
+      load_gradefile if superscore
+
+      if superscore && @tests.all? { |test| test[:score] == test[:max] }
+        @done = true
+        return
+      end
 
       self.status = :ungraded
-      FileUtils.rm_f [buildlog, testlog]
+      archive_logs superscore
       @fs.jail.reset
       @fs.jail.init
     end
 
     def stage
+      return if @done
       @assignment.manifest[:provided].each do |entry|
         @fs.jail.stage entry[:path], entry[:name]
       end
@@ -131,35 +150,86 @@ module KGrader
     end
 
     def build
+      return if @done
       @assignment.build_steps.each do |command|
         return build_failure unless @fs.jail.exec command, buildlog
       end
     end
 
     def test
-      return if @failure
-      @tests.each do |test|
-        test[:score] = @fs.jail.run_test test[:script], testlog do |comment|
-          @comments.push "#{test[:name]}: #{comment}"
-        end
-      end
+      return if @done
+      @tests.each { |test| run_test test }
     end
 
     def save
-      File.write gradefile, generate_report
-      FileUtils.touch pendingfile
+      if @changed
+        File.write gradefile, generate_gradefile
+        File.write gradereport, generate_report
+        FileUtils.touch pendingfile
+      end
     end
 
     def grade_post
       self.status = :graded
       @fs.jail.reset
-      @summary = generate_summary unless @summary
+      @summary = generate_summary
+    end
+
+    # -------------------------------------------------------------------------
+
+    def archive_logs(superscore)
+      logs = [buildlog, testlog]
+      if superscore
+        logs.select { |log| File.exists? log }.each do |log|
+          File.rename log, "#{log}.#{Time.now.strftime('%Y%m%d%H%M%S')}"
+        end
+      else
+        FileUtils.rm_f logs
+      end
     end
 
     def build_failure
-      @failure = true
-      @comments.push "failed to compile"
-      @summary = "#{format_points 0, max_score}: failed to compile"
+      @done = true
+      @failed = true
+    end
+
+    def run_test(test)
+      return if test[:score] == text[:max]  # Can't superscore the max score
+
+      test[:depends].each do |depname|
+        dep = @tests.find { |t| t[:name] == depname }
+        if !dep.nil? && dep[:score] == 0
+          test[:comments] = [test[:depfail]]
+          return
+        end
+      end
+
+      score, comments = @fs.jail.run_test test[:script], testlog
+      if score > test[:score]
+        test[:score] = score
+        test[:comments] = comments
+        @changed = true
+      end
+    end
+
+    def load_gradefile
+      begin
+        data = @fs.load gradefile
+      rescue FilesystemError
+        return
+      end
+      data.each do |name, fields|
+        test = @tests.find { |t| t[:name] == name }
+        test[:score], test[:comments] = fields
+      end
+    end
+
+    def generate_gradefile
+      data = @tests.inject({}) do |hash, test|
+        hash[test[:name]] = [test[:score], test[:comments]]
+        hash
+      end
+      JSON.generate data
     end
 
     def generate_report
@@ -175,7 +245,7 @@ module KGrader
       ]
       version = KGrader.version
       metadata.push "grader version:  #{version}" if version
-      metadata = metadata.join("\n")
+      metadata = metadata.join "\n"
 
       tests = "tests:\n" + @tests.map do |test|
         score = format_points(test[:score], test[:max], max_score)
@@ -183,24 +253,22 @@ module KGrader
       end.join("\n")
 
       total = justify_both "total:", format_points(score, max_score)
-
-      all_comments = (@comments + @assignment.extra_comments)
-      if all_comments
-        comments = "comments:\n" + all_comments.map do |cmt|
-          "    - #{cmt}\n"
-        end.join
-      else
-        comments = ""
-      end
+      comments = generate_comments
 
       [header, hr2, metadata, hr1, tests, hr1, total, hr1, comments].join "\n"
     end
 
     def generate_summary
-      tests = @tests.map do |test|
-        "#{test[:score].to_s.rjust get_span(test[:max])}/#{test[:max]}"
-      end.join ', '
-      "#{format_points score, max_score}: #{tests}"
+      if !@changed
+        line = "no change"
+      elsif @failed
+        line = "failed to compile"
+      else
+        line = @tests.map do |test|
+          "#{test[:score].to_s.rjust get_span(test[:max])}/#{test[:max]}"
+        end.join ', '
+      end
+      "#{format_points score, max_score}: #{line}"
     end
 
     def score
@@ -211,6 +279,18 @@ module KGrader
       @tests.reduce(0) { |sum, t| sum + t[:max] }
     end
 
+    def generate_comments
+      comments = []
+      comments.push "failed to compile" if @failed
+      @tests.each do |test|
+        test[:comments].each { |cmt| comments.push "#{test[:name]}: #{cmt}" }
+      end
+      comments.concat @assignment.extra_comments
+
+      return "" if comments.empty?
+      "comments:\n" + comments.map { |cmt| "    - #{cmt}\n" }.join
+    end
+
     def format_points(score, max, span_max = nil)
       percent = (score.to_f * 100 / max).round.to_s.rjust 3
       span = get_span(span_max || max)

lib/kgrader/task.rb

@@ -15,10 +15,14 @@ module KGrader
       due = options.fetch(:due, Time.now)
       fetch = options.fetch(:fetch, true)
       regrade = options.fetch(:regrade, false)
+      superscore = options.fetch(:superscore, false)
 
       if options.include?(:due) && !fetch
         raise TaskError, "can't set a new due date without fetching"
       end
+      if regrade && superscore
+        raise TaskError, "can't regrade and superscore at the same time"
+      end
 
       prepare
       subtask 'setup' do |sub|
@@ -34,10 +38,10 @@ module KGrader
       subtask 'grade' do |sub|
         if sub.status == :init || sub.status == :fetching
           next 'skip (need to fetch first)'
-        elsif sub.status == :graded && !regrade
+        elsif sub.status == :graded && !regrade && !superscore
           next
         else
-          sub.grade
+          sub.grade superscore
         end
       end
     end

rakefile

@@ -26,6 +26,7 @@ task :help do
 - rake roster <course> <rosterfile> [<semester>]
 - rake grade  <course> <assignment> [semester=<...>] [students=<...>]
               [due=<...>] [fetch=<yes/no>] [regrade=<yes/no>]
+              [superscore=<yes/no>]
 - rake commit <course> <assignment> [semester=<...>] [students=<...>]
 - rake clean
 - rake clobber}
@@ -44,7 +45,7 @@ end
 task :grade do
   course, assignment, options = parse_args 2..2,
     { :semester => :string, :students => :array, :due => :time,
-      :fetch => :bool, :regrade => :bool }
+      :fetch => :bool, :regrade => :bool, :superscore => :bool }
   semester, students = options[:semester], options[:students]
   run { |cli| cli.grade course, semester, assignment, students, options }
 end

spec/cs241h

@@ -1 +1 @@
-Subproject commit 55821824118089c1cbd69470702751b03d8552a7
+Subproject commit bac3b71f3215b1193a2c9dc17372aff375f888cc