diff --git a/README.md b/README.md index 70f213b..85e30c7 100644 --- a/README.md +++ b/README.md @@ -95,5 +95,10 @@ repo has changed since the last run. To forcibly regrade: rake grade cs123 mp1 regrade=yes +You can enable superscoring as well, which will re-run tests that a student +failed in an attempt to see if they can get a higher grade: + + rake grade cs123 mp1 superscore=yes + You can combine these arguments in any meaningful way. `semester` and `students` are also available options for `commit`. diff --git a/lib/kgrader/assignment.rb b/lib/kgrader/assignment.rb index cf12807..ae00cd3 100644 --- a/lib/kgrader/assignment.rb +++ b/lib/kgrader/assignment.rb @@ -30,8 +30,13 @@ module KGrader def tests @tests ||= @config['grade'].map do |it| - script = File.join @root, it.keys.first + ".rb" - { :name => it.keys.first, :script => script, :max => it.values.first } + name = it.keys.first + opts = it.values.first + script = File.join @root, name + '.rb' + depends = *opts['depends'] || [] + depfail = opts['depfail'] || "depends on #{name}" + { :name => name, :script => script, :max => opts['points'], + :depends => depends, :depfail => depfail } end end @@ -47,7 +52,8 @@ module KGrader end def extra_comments - @config['commit']['comments'] || [] + conf = @config['commit'] + return *conf['comments'] || conf['comment'] || [] end private diff --git a/lib/kgrader/filesystem.rb b/lib/kgrader/filesystem.rb index 58f7e91..b4228e8 100644 --- a/lib/kgrader/filesystem.rb +++ b/lib/kgrader/filesystem.rb @@ -80,7 +80,7 @@ module KGrader File.write fn, "ungraded" if File.read(fn) == "graded" end FileUtils.rm_f Dir[File.join desk_dir, '*', '*', '*', '*', 'pending'] - FileUtils.rm_f Dir[File.join desk_dir, '*', '*', '*', '*', '*.log'] + FileUtils.rm_f Dir[File.join desk_dir, '*', '*', '*', '*', '*.log*'] end # ------------------------------------------------------------------------- diff --git a/lib/kgrader/jail.rb b/lib/kgrader/jail.rb index ac8a803..8fe2db2 100644 --- a/lib/kgrader/jail.rb +++ b/lib/kgrader/jail.rb @@ -39,10 +39,10 @@ module KGrader [grade_wr, cmt_wr].each &:close Process.waitpid pid, 0 - cmt_rd.read.split("\n").each { |cmt| yield cmt } if block_given? grade = grade_rd.read.strip.to_i + comments = cmt_rd.read.split("\n") [grade_rd, cmt_rd].each &:close - grade + return grade, comments end private diff --git a/lib/kgrader/submission.rb b/lib/kgrader/submission.rb index 273f1d6..fd37264 100644 --- a/lib/kgrader/submission.rb +++ b/lib/kgrader/submission.rb @@ -48,8 +48,8 @@ module KGrader nil end - def grade - grade_prep + def grade(superscore = false) + grade_prep superscore stage build test @@ -61,13 +61,15 @@ module KGrader def commit if status == :graded && File.exists?(pendingfile) message = @assignment.commit_message @student - FileUtils.cp gradefile, File.join(repo, @assignment.report) + FileUtils.cp gradereport, File.join(repo, @assignment.report) @course.backend.commit repo, message, @assignment.report FileUtils.rm pendingfile end nil end + # ------------------------------------------------------------------------- + private def repo File.join @root, 'repo' @@ -78,6 +80,10 @@ module KGrader end def gradefile + File.join @root, 'grade.json' + end + + def gradereport File.join @root, 'grade.txt' end @@ -109,19 +115,32 @@ module KGrader rev end - def grade_prep - @failure = false - @comments = [] + # ------------------------------------------------------------------------- + + def grade_prep(superscore) + @done = false + @failed = false + @changed = !superscore || self.status == :ungraded @summary = nil - @tests = @assignment.tests.clone.each { |test| test[:score] = 0 } + @tests = @assignment.tests.clone.each do |test| + test[:score] = 0 + test[:comments] = [] + end + load_gradefile if superscore + + if superscore && @tests.all? { |test| test[:score] == test[:max] } + @done = true + return + end self.status = :ungraded - FileUtils.rm_f [buildlog, testlog] + archive_logs superscore @fs.jail.reset @fs.jail.init end def stage + return if @done @assignment.manifest[:provided].each do |entry| @fs.jail.stage entry[:path], entry[:name] end @@ -131,35 +150,86 @@ module KGrader end def build + return if @done @assignment.build_steps.each do |command| return build_failure unless @fs.jail.exec command, buildlog end end def test - return if @failure - @tests.each do |test| - test[:score] = @fs.jail.run_test test[:script], testlog do |comment| - @comments.push "#{test[:name]}: #{comment}" - end - end + return if @done + @tests.each { |test| run_test test } end def save - File.write gradefile, generate_report - FileUtils.touch pendingfile + if @changed + File.write gradefile, generate_gradefile + File.write gradereport, generate_report + FileUtils.touch pendingfile + end end def grade_post self.status = :graded @fs.jail.reset - @summary = generate_summary unless @summary + @summary = generate_summary + end + + # ------------------------------------------------------------------------- + + def archive_logs(superscore) + logs = [buildlog, testlog] + if superscore + logs.select { |log| File.exists? log }.each do |log| + File.rename log, "#{log}.#{Time.now.strftime('%Y%m%d%H%M%S')}" + end + else + FileUtils.rm_f logs + end end def build_failure - @failure = true - @comments.push "failed to compile" - @summary = "#{format_points 0, max_score}: failed to compile" + @done = true + @failed = true + end + + def run_test(test) + return if test[:score] == text[:max] # Can't superscore the max score + + test[:depends].each do |depname| + dep = @tests.find { |t| t[:name] == depname } + if !dep.nil? && dep[:score] == 0 + test[:comments] = [test[:depfail]] + return + end + end + + score, comments = @fs.jail.run_test test[:script], testlog + if score > test[:score] + test[:score] = score + test[:comments] = comments + @changed = true + end + end + + def load_gradefile + begin + data = @fs.load gradefile + rescue FilesystemError + return + end + data.each do |name, fields| + test = @tests.find { |t| t[:name] == name } + test[:score], test[:comments] = fields + end + end + + def generate_gradefile + data = @tests.inject({}) do |hash, test| + hash[test[:name]] = [test[:score], test[:comments]] + hash + end + JSON.generate data end def generate_report @@ -175,7 +245,7 @@ module KGrader ] version = KGrader.version metadata.push "grader version: #{version}" if version - metadata = metadata.join("\n") + metadata = metadata.join "\n" tests = "tests:\n" + @tests.map do |test| score = format_points(test[:score], test[:max], max_score) @@ -183,24 +253,22 @@ module KGrader end.join("\n") total = justify_both "total:", format_points(score, max_score) - - all_comments = (@comments + @assignment.extra_comments) - if all_comments - comments = "comments:\n" + all_comments.map do |cmt| - " - #{cmt}\n" - end.join - else - comments = "" - end + comments = generate_comments [header, hr2, metadata, hr1, tests, hr1, total, hr1, comments].join "\n" end def generate_summary - tests = @tests.map do |test| - "#{test[:score].to_s.rjust get_span(test[:max])}/#{test[:max]}" - end.join ', ' - "#{format_points score, max_score}: #{tests}" + if !@changed + line = "no change" + elsif @failed + line = "failed to compile" + else + line = @tests.map do |test| + "#{test[:score].to_s.rjust get_span(test[:max])}/#{test[:max]}" + end.join ', ' + end + "#{format_points score, max_score}: #{line}" end def score @@ -211,6 +279,18 @@ module KGrader @tests.reduce(0) { |sum, t| sum + t[:max] } end + def generate_comments + comments = [] + comments.push "failed to compile" if @failed + @tests.each do |test| + test[:comments].each { |cmt| comments.push "#{test[:name]}: #{cmt}" } + end + comments.concat @assignment.extra_comments + + return "" if comments.empty? + "comments:\n" + comments.map { |cmt| " - #{cmt}\n" }.join + end + def format_points(score, max, span_max = nil) percent = (score.to_f * 100 / max).round.to_s.rjust 3 span = get_span(span_max || max) diff --git a/lib/kgrader/task.rb b/lib/kgrader/task.rb index fc69a1d..21675d7 100644 --- a/lib/kgrader/task.rb +++ b/lib/kgrader/task.rb @@ -15,10 +15,14 @@ module KGrader due = options.fetch(:due, Time.now) fetch = options.fetch(:fetch, true) regrade = options.fetch(:regrade, false) + superscore = options.fetch(:superscore, false) if options.include?(:due) && !fetch raise TaskError, "can't set a new due date without fetching" end + if regrade && superscore + raise TaskError, "can't regrade and superscore at the same time" + end prepare subtask 'setup' do |sub| @@ -34,10 +38,10 @@ module KGrader subtask 'grade' do |sub| if sub.status == :init || sub.status == :fetching next 'skip (need to fetch first)' - elsif sub.status == :graded && !regrade + elsif sub.status == :graded && !regrade && !superscore next else - sub.grade + sub.grade superscore end end end diff --git a/rakefile b/rakefile index 65157ab..f34ad94 100644 --- a/rakefile +++ b/rakefile @@ -26,6 +26,7 @@ task :help do - rake roster [] - rake grade [semester=<...>] [students=<...>] [due=<...>] [fetch=] [regrade=] + [superscore=] - rake commit [semester=<...>] [students=<...>] - rake clean - rake clobber} @@ -44,7 +45,7 @@ end task :grade do course, assignment, options = parse_args 2..2, { :semester => :string, :students => :array, :due => :time, - :fetch => :bool, :regrade => :bool } + :fetch => :bool, :regrade => :bool, :superscore => :bool } semester, students = options[:semester], options[:students] run { |cli| cli.grade course, semester, assignment, students, options } end diff --git a/spec/cs241h b/spec/cs241h index 5582182..bac3b71 160000 --- a/spec/cs241h +++ b/spec/cs241h @@ -1 +1 @@ -Subproject commit 55821824118089c1cbd69470702751b03d8552a7 +Subproject commit bac3b71f3215b1193a2c9dc17372aff375f888cc