Browse Source

Add support for superscoring.

master
Ben Kurtovic 8 years ago
parent
commit
77d69d92a0
8 changed files with 140 additions and 44 deletions
  1. +5
    -0
      README.md
  2. +9
    -3
      lib/kgrader/assignment.rb
  3. +1
    -1
      lib/kgrader/filesystem.rb
  4. +2
    -2
      lib/kgrader/jail.rb
  5. +114
    -34
      lib/kgrader/submission.rb
  6. +6
    -2
      lib/kgrader/task.rb
  7. +2
    -1
      rakefile
  8. +1
    -1
      spec/cs241h

+ 5
- 0
README.md View File

@@ -95,5 +95,10 @@ repo has changed since the last run. To forcibly regrade:

rake grade cs123 mp1 regrade=yes

You can enable superscoring as well, which will re-run tests that a student
failed in an attempt to see if they can get a higher grade:

rake grade cs123 mp1 superscore=yes

You can combine these arguments in any meaningful way. `semester` and
`students` are also available options for `commit`.

+ 9
- 3
lib/kgrader/assignment.rb View File

@@ -30,8 +30,13 @@ module KGrader

def tests
@tests ||= @config['grade'].map do |it|
script = File.join @root, it.keys.first + ".rb"
{ :name => it.keys.first, :script => script, :max => it.values.first }
name = it.keys.first
opts = it.values.first
script = File.join @root, name + '.rb'
depends = *opts['depends'] || []
depfail = opts['depfail'] || "depends on #{name}"
{ :name => name, :script => script, :max => opts['points'],
:depends => depends, :depfail => depfail }
end
end

@@ -47,7 +52,8 @@ module KGrader
end

def extra_comments
@config['commit']['comments'] || []
conf = @config['commit']
return *conf['comments'] || conf['comment'] || []
end

private


+ 1
- 1
lib/kgrader/filesystem.rb View File

@@ -80,7 +80,7 @@ module KGrader
File.write fn, "ungraded" if File.read(fn) == "graded"
end
FileUtils.rm_f Dir[File.join desk_dir, '*', '*', '*', '*', 'pending']
FileUtils.rm_f Dir[File.join desk_dir, '*', '*', '*', '*', '*.log']
FileUtils.rm_f Dir[File.join desk_dir, '*', '*', '*', '*', '*.log*']
end

# -------------------------------------------------------------------------


+ 2
- 2
lib/kgrader/jail.rb View File

@@ -39,10 +39,10 @@ module KGrader
[grade_wr, cmt_wr].each &:close
Process.waitpid pid, 0

cmt_rd.read.split("\n").each { |cmt| yield cmt } if block_given?
grade = grade_rd.read.strip.to_i
comments = cmt_rd.read.split("\n")
[grade_rd, cmt_rd].each &:close
grade
return grade, comments
end

private


+ 114
- 34
lib/kgrader/submission.rb View File

@@ -48,8 +48,8 @@ module KGrader
nil
end

def grade
grade_prep
def grade(superscore = false)
grade_prep superscore
stage
build
test
@@ -61,13 +61,15 @@ module KGrader
def commit
if status == :graded && File.exists?(pendingfile)
message = @assignment.commit_message @student
FileUtils.cp gradefile, File.join(repo, @assignment.report)
FileUtils.cp gradereport, File.join(repo, @assignment.report)
@course.backend.commit repo, message, @assignment.report
FileUtils.rm pendingfile
end
nil
end

# -------------------------------------------------------------------------

private
def repo
File.join @root, 'repo'
@@ -78,6 +80,10 @@ module KGrader
end

def gradefile
File.join @root, 'grade.json'
end

def gradereport
File.join @root, 'grade.txt'
end

@@ -109,19 +115,32 @@ module KGrader
rev
end

def grade_prep
@failure = false
@comments = []
# -------------------------------------------------------------------------

def grade_prep(superscore)
@done = false
@failed = false
@changed = !superscore || self.status == :ungraded
@summary = nil
@tests = @assignment.tests.clone.each { |test| test[:score] = 0 }
@tests = @assignment.tests.clone.each do |test|
test[:score] = 0
test[:comments] = []
end
load_gradefile if superscore

if superscore && @tests.all? { |test| test[:score] == test[:max] }
@done = true
return
end

self.status = :ungraded
FileUtils.rm_f [buildlog, testlog]
archive_logs superscore
@fs.jail.reset
@fs.jail.init
end

def stage
return if @done
@assignment.manifest[:provided].each do |entry|
@fs.jail.stage entry[:path], entry[:name]
end
@@ -131,35 +150,86 @@ module KGrader
end

def build
return if @done
@assignment.build_steps.each do |command|
return build_failure unless @fs.jail.exec command, buildlog
end
end

def test
return if @failure
@tests.each do |test|
test[:score] = @fs.jail.run_test test[:script], testlog do |comment|
@comments.push "#{test[:name]}: #{comment}"
end
end
return if @done
@tests.each { |test| run_test test }
end

def save
File.write gradefile, generate_report
FileUtils.touch pendingfile
if @changed
File.write gradefile, generate_gradefile
File.write gradereport, generate_report
FileUtils.touch pendingfile
end
end

def grade_post
self.status = :graded
@fs.jail.reset
@summary = generate_summary unless @summary
@summary = generate_summary
end

# -------------------------------------------------------------------------

def archive_logs(superscore)
logs = [buildlog, testlog]
if superscore
logs.select { |log| File.exists? log }.each do |log|
File.rename log, "#{log}.#{Time.now.strftime('%Y%m%d%H%M%S')}"
end
else
FileUtils.rm_f logs
end
end

def build_failure
@failure = true
@comments.push "failed to compile"
@summary = "#{format_points 0, max_score}: failed to compile"
@done = true
@failed = true
end

def run_test(test)
return if test[:score] == text[:max] # Can't superscore the max score

test[:depends].each do |depname|
dep = @tests.find { |t| t[:name] == depname }
if !dep.nil? && dep[:score] == 0
test[:comments] = [test[:depfail]]
return
end
end

score, comments = @fs.jail.run_test test[:script], testlog
if score > test[:score]
test[:score] = score
test[:comments] = comments
@changed = true
end
end

def load_gradefile
begin
data = @fs.load gradefile
rescue FilesystemError
return
end
data.each do |name, fields|
test = @tests.find { |t| t[:name] == name }
test[:score], test[:comments] = fields
end
end

def generate_gradefile
data = @tests.inject({}) do |hash, test|
hash[test[:name]] = [test[:score], test[:comments]]
hash
end
JSON.generate data
end

def generate_report
@@ -175,7 +245,7 @@ module KGrader
]
version = KGrader.version
metadata.push "grader version: #{version}" if version
metadata = metadata.join("\n")
metadata = metadata.join "\n"

tests = "tests:\n" + @tests.map do |test|
score = format_points(test[:score], test[:max], max_score)
@@ -183,24 +253,22 @@ module KGrader
end.join("\n")

total = justify_both "total:", format_points(score, max_score)

all_comments = (@comments + @assignment.extra_comments)
if all_comments
comments = "comments:\n" + all_comments.map do |cmt|
" - #{cmt}\n"
end.join
else
comments = ""
end
comments = generate_comments

[header, hr2, metadata, hr1, tests, hr1, total, hr1, comments].join "\n"
end

def generate_summary
tests = @tests.map do |test|
"#{test[:score].to_s.rjust get_span(test[:max])}/#{test[:max]}"
end.join ', '
"#{format_points score, max_score}: #{tests}"
if !@changed
line = "no change"
elsif @failed
line = "failed to compile"
else
line = @tests.map do |test|
"#{test[:score].to_s.rjust get_span(test[:max])}/#{test[:max]}"
end.join ', '
end
"#{format_points score, max_score}: #{line}"
end

def score
@@ -211,6 +279,18 @@ module KGrader
@tests.reduce(0) { |sum, t| sum + t[:max] }
end

def generate_comments
comments = []
comments.push "failed to compile" if @failed
@tests.each do |test|
test[:comments].each { |cmt| comments.push "#{test[:name]}: #{cmt}" }
end
comments.concat @assignment.extra_comments

return "" if comments.empty?
"comments:\n" + comments.map { |cmt| " - #{cmt}\n" }.join
end

def format_points(score, max, span_max = nil)
percent = (score.to_f * 100 / max).round.to_s.rjust 3
span = get_span(span_max || max)


+ 6
- 2
lib/kgrader/task.rb View File

@@ -15,10 +15,14 @@ module KGrader
due = options.fetch(:due, Time.now)
fetch = options.fetch(:fetch, true)
regrade = options.fetch(:regrade, false)
superscore = options.fetch(:superscore, false)

if options.include?(:due) && !fetch
raise TaskError, "can't set a new due date without fetching"
end
if regrade && superscore
raise TaskError, "can't regrade and superscore at the same time"
end

prepare
subtask 'setup' do |sub|
@@ -34,10 +38,10 @@ module KGrader
subtask 'grade' do |sub|
if sub.status == :init || sub.status == :fetching
next 'skip (need to fetch first)'
elsif sub.status == :graded && !regrade
elsif sub.status == :graded && !regrade && !superscore
next
else
sub.grade
sub.grade superscore
end
end
end


+ 2
- 1
rakefile View File

@@ -26,6 +26,7 @@ task :help do
- rake roster <course> <rosterfile> [<semester>]
- rake grade <course> <assignment> [semester=<...>] [students=<...>]
[due=<...>] [fetch=<yes/no>] [regrade=<yes/no>]
[superscore=<yes/no>]
- rake commit <course> <assignment> [semester=<...>] [students=<...>]
- rake clean
- rake clobber}
@@ -44,7 +45,7 @@ end
task :grade do
course, assignment, options = parse_args 2..2,
{ :semester => :string, :students => :array, :due => :time,
:fetch => :bool, :regrade => :bool }
:fetch => :bool, :regrade => :bool, :superscore => :bool }
semester, students = options[:semester], options[:students]
run { |cli| cli.grade course, semester, assignment, students, options }
end


+ 1
- 1
spec/cs241h

@@ -1 +1 @@
Subproject commit 55821824118089c1cbd69470702751b03d8552a7
Subproject commit bac3b71f3215b1193a2c9dc17372aff375f888cc

Loading…
Cancel
Save