Admin csv import

.tool-versions

@@ -0,0 +1 @@
+ruby 4.0.6

Gemfile

@@ -42,6 +42,8 @@ gem "thruster", require: false
 gem "image_processing", "~> 2.1"
 gem "ruby-vips", "~> 2.0"
 
+gem "csv"
+
 group :development, :test do
   # See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem
   gem "debug", platforms: %i[ mri windows ], require: "debug/prelude"

Gemfile.lock

@@ -114,6 +114,7 @@ GEM
     cruise (0.2.0-arm64-darwin)
     cruise (0.2.0-x86_64-linux-gnu)
     cruise (0.2.0-x86_64-linux-musl)
+    csv (3.3.6)
     date (3.5.1)
     debug (1.11.1)
       irb (~> 1.10)
@@ -434,6 +435,7 @@ DEPENDENCIES
   brakeman
   bundler-audit
   capybara
+  csv
   debug
   faker (~> 3.8)
   herb
@@ -496,6 +498,7 @@ CHECKSUMS
   cruise (0.2.0-arm64-darwin) sha256=fb3e9b265868e077dd754b4a430ab14d8abb209281c61c906b74f326b797a10a
   cruise (0.2.0-x86_64-linux-gnu) sha256=3d16f6a6a3409f2cae5dbdb3fb322389fa61a6a18d80a2df50fc4c807f1f0fd9
   cruise (0.2.0-x86_64-linux-musl) sha256=2be5c5f2fb474ff5a4f42e3421dbb27fb439008a53697f7a6a3d2f3ec2de711c
+  csv (3.3.6) sha256=aba61e7e507a66f03d45cb1f3c4b6359861c3504038b422962875dce099e4456
   date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0
   debug (1.11.1) sha256=2e0b0ac6119f2207a6f8ac7d4a73ca8eb4e440f64da0a3136c30343146e952b6
   dotenv (3.2.0) sha256=e375b83121ea7ca4ce20f214740076129ab8514cd81378161f11c03853fe619d

app/controllers/admin/csv_controller.rb

@@ -0,0 +1,67 @@
+require "csv"
+
+class Admin::CsvController < AdminController
+  class CSVHeadersError < StandardError; end
+  class StudentBulkImportError < StandardError; end
+  class ClassroomBulkImportError < StandardError; end
+
+  IMPORT_ERRORS = [
+    CSVHeadersError,
+    StudentBulkImportError,
+    ClassroomBulkImportError,
+    CSV::MalformedCSVError,
+    StudentCsvImporter::InvalidClassroomError,
+    StudentCsvImporter::InvalidStudentError,
+    StudentCsvImporter::InvalidTeacherError
+  ].freeze
+
+  CSV_HEADERS = [ "Student First Name", "Student Last Name", "Grade Level", "Class Name", "Teacher", "Teacher Email",
+                 "Program", "Program Level" ].freeze
+
+  before_action :set_school
+
+  def download
+    csv_data = CSV.generate do |csv|
+      csv << CSV_HEADERS
+    end
+
+    send_data csv_data,
+              filename: "students-#{Date.today}.csv",
+              type: "text/csv; charset=utf-8",
+              disposition: "attachment"
+  end
+
+  def import
+    csv_file = params[:file]
+    return redirect_to school_students_path(@school), alert: "Choose a CSV file to import." if csv_file.blank?
+
+    csv = CSV.read(csv_file.path, headers: true)
+    validate_rows!(csv)
+    StudentCsvImporter.new(csv: csv, school_id: @school.id).import
+
+    redirect_to school_students_path(@school), notice: "Students were successfully imported."
+  rescue *IMPORT_ERRORS => error
+    redirect_to school_students_path(@school), alert: error.message
+  end
+
+  private
+    def set_school
+      @school = School.find(params.expect(:school_id))
+    end
+
+    def validate_rows!(csv)
+      raise CSVHeadersError, "Headers must match CSV headers" unless CSV_HEADERS == csv.headers
+
+      csv.each do |row|
+        raise CSVHeadersError, "Row must have same number of columns as headers" unless CSV_HEADERS.length == row.length
+
+        if @school.students.exists?(first_name: row["Student First Name"], last_name: row["Student Last Name"])
+          raise StudentBulkImportError, "Student: #{row["Student First Name"]} #{row["Student Last Name"]} already exists"
+        end
+
+        if @school.classrooms.exists?(name: row["Class Name"])
+          raise ClassroomBulkImportError, "Classroom: #{row["Class Name"]} already exists"
+        end
+      end
+    end
+end

app/services/student_csv_importer.rb

@@ -0,0 +1,116 @@
+
+
+class StudentCsvImporter
+  class InvalidClassroomError < StandardError; end
+  class InvalidStudentError < StandardError; end
+  class InvalidTeacherError < StandardError; end
+
+  def initialize(csv:, school_id:)
+    @csv = csv
+    @school_id = school_id
+    @error_messages = {
+      classrooms: {},
+      teachers: {},
+      students: {}
+    }
+    @classrooms = {}
+    @teachers = {}
+    @students = []
+    @row_indexes = {}
+  end
+
+
+  def import
+    @csv.each_with_index do |row, index|
+      next if row.blank?
+
+      teacher = find_or_build_teacher(row)
+      classroom = find_or_build_classroom(row, teacher)
+
+      student = Student.new(
+        first_name: row["Student First Name"],
+        last_name: row["Student Last Name"],
+        grade_level: row["Grade Level"],
+        school_id: @school_id,
+        classroom: classroom
+      )
+
+      collect_errors(:teachers, teacher, index)
+      collect_errors(:classrooms, classroom, index)
+      collect_errors(:students, student, index)
+
+      @students << student
+    end
+
+    save_records
+  end
+
+
+
+private
+
+  def save_records
+    ActiveRecord::Base.transaction do
+      raise_validation_errors!
+
+      save_all(@teachers.values, InvalidTeacherError)
+      save_all(@classrooms.values, InvalidClassroomError)
+      save_all(@students, InvalidStudentError)
+
+      create_program_associations
+    end
+  end
+
+  # A uniqueness collision between two rows of the same file survives `valid?`,
+  # because neither record is persisted yet. Re-raise it in the shape the
+  # validation errors already use so callers only have to handle one thing.
+  def save_all(records, error_class)
+    records.each do |record|
+      record.save!
+    rescue ActiveRecord::RecordInvalid => error
+      raise error_class, { @row_indexes[record.object_id] => error.record.errors.full_messages }.to_s
+    end
+  end
+
+  def create_program_associations
+    @csv.each do |row|
+      teacher = @teachers[row["Teacher"]]
+      classroom = @classrooms[[ teacher.name, row["Class Name"] ]]
+
+      program = Program.find_or_create_by!(name: row["Program"])
+
+      classroom.classroom_programs.create_or_find_by!(program: program, level: row["Program Level"])
+    end
+  end
+  def collect_errors(type, record, index)
+    @row_indexes[record.object_id] ||= index
+
+    return if record.valid?
+
+    @error_messages[type][index] = record.errors.full_messages
+  end
+
+  def raise_validation_errors!
+    raise InvalidClassroomError, @error_messages[:classrooms].to_s if @error_messages[:classrooms].any?
+    raise InvalidStudentError, @error_messages[:students].to_s if @error_messages[:students].any?
+    raise InvalidTeacherError, @error_messages[:teachers].to_s if @error_messages[:teachers].any?
+  end
+
+  def find_or_build_teacher(row)
+    @teachers[row["Teacher"]] ||= Teacher.new(
+      name: row["Teacher"],
+      email: row["Teacher Email"],
+      school_id: @school_id
+    )
+  end
+
+  def find_or_build_classroom(row, teacher)
+    key = [ teacher.name, row["Class Name"] ]
+
+    @classrooms[key] ||= Classroom.new(
+      school_id: @school_id,
+      name: row["Class Name"],
+      teacher: teacher
+    )
+  end
+end

config/routes.rb

@@ -25,6 +25,11 @@
         member { get :schedule }
       end
       resources :teachers, shallow: true, except: [ :show ]
+
+      scope module: :admin do
+        get :csv_template, to: "csv#download"
+        post :csv_import, to: "csv#import"
+      end
     end
     resources :content_modules, except: [ :show ] do
       resources :links, shallow: true, except: %i[index show]

test/controllers/admin/csv_controller_test.rb

@@ -0,0 +1,152 @@
+require "test_helper"
+require "csv"
+
+class Admin::CsvControllerTest < ActionDispatch::IntegrationTest
+  setup do
+    @school = schools(:one)
+    sign_in_as users(:admin)
+  end
+
+  test "should redirect the template download when not authenticated" do
+    sign_out
+
+    get school_csv_template_url(@school)
+
+    assert_redirected_to new_session_path
+  end
+
+  test "should redirect the import and write nothing when not authenticated" do
+    sign_out
+
+    assert_no_difference "Student.count" do
+      post school_csv_import_url(@school), params: { file: csv_upload("students.csv") }
+    end
+
+    assert_redirected_to new_session_path
+  end
+
+  test "should download the student import template" do
+    get school_csv_template_url(@school)
+
+    assert_response :success
+    assert_equal "text/csv", response.media_type
+    assert_match(/attachment/, response.headers["Content-Disposition"])
+    assert_equal Admin::CsvController::CSV_HEADERS, CSV.parse(response.body).first
+  end
+
+  test "should import students, teachers, classrooms and program enrollments" do
+    assert_difference -> { @school.students.count } => 3,
+                      -> { @school.teachers.count } => 2,
+                      -> { @school.classrooms.count } => 2,
+                      -> { ClassroomProgram.count } => 2 do
+      post school_csv_import_url(@school), params: { file: csv_upload("students.csv") }
+    end
+
+    assert_redirected_to school_students_path(@school)
+    assert_equal "Students were successfully imported.", flash[:notice]
+
+    room_a = @school.classrooms.find_by(name: "Room A")
+    assert_equal "Nina Simone", room_a.teacher.name
+    assert_equal [ "Alan Turing", "Katherine Johnson" ], room_a.students.map(&:full_name).sort
+    assert_equal [ programs(:kyh) ], room_a.programs.to_a
+    assert_equal [ "basic" ], room_a.classroom_programs.map(&:level)
+  end
+
+  test "should import into the school named in the route" do
+    other_school = schools(:two)
+
+    post school_csv_import_url(other_school), params: { file: csv_upload("students.csv") }
+
+    assert_redirected_to school_students_path(other_school)
+    assert other_school.students.exists?(first_name: "Alan", last_name: "Turing")
+    assert_not @school.students.exists?(first_name: "Alan", last_name: "Turing")
+  end
+
+  test "should reject a student who already belongs to the school" do
+    assert_no_difference "Student.count" do
+      post school_csv_import_url(@school), params: { file: csv_upload("students_existing_student.csv") }
+    end
+
+    assert_redirected_to school_students_path(@school)
+    assert_equal "Student: Ada Lovelace already exists", flash[:alert]
+  end
+
+  test "should reject a classroom that already belongs to the school" do
+    assert_no_difference "Classroom.count" do
+      post school_csv_import_url(@school), params: { file: csv_upload("students_existing_classroom_name.csv") }
+    end
+
+    assert_redirected_to school_students_path(@school)
+    assert_equal "Classroom: Classroom 1 already exists", flash[:alert]
+  end
+
+  test "should allow a classroom name that is only taken at another school" do
+    other_school = schools(:two)
+
+    assert_difference -> { other_school.classrooms.count }, 1 do
+      post school_csv_import_url(other_school), params: { file: csv_upload("students_existing_classroom_name.csv") }
+    end
+
+    assert_equal "Students were successfully imported.", flash[:notice]
+  end
+
+  test "should reject headers that do not match the template" do
+    assert_no_difference "Student.count" do
+      post school_csv_import_url(@school), params: { file: csv_upload("students_bad_headers.csv") }
+    end
+
+    assert_redirected_to school_students_path(@school)
+    assert_equal "Headers must match CSV headers", flash[:alert]
+  end
+
+  # An extra field widens the parsed header row, so this trips the header check
+  # rather than the per-row column count check.
+  test "should reject a row with more columns than the template" do
+    assert_no_difference "Student.count" do
+      post school_csv_import_url(@school), params: { file: csv_upload("students_extra_column.csv") }
+    end
+
+    assert_redirected_to school_students_path(@school)
+    assert_equal "Headers must match CSV headers", flash[:alert]
+  end
+
+  test "should report two rows giving different teachers the same email" do
+    assert_no_difference [ "Student.count", "Teacher.count" ] do
+      post school_csv_import_url(@school), params: { file: csv_upload("students_duplicate_teacher_email.csv") }
+    end
+
+    assert_redirected_to school_students_path(@school)
+    assert_match "Email has already been taken", flash[:alert]
+  end
+
+  test "should report validation errors and write nothing" do
+    assert_no_difference [ "Student.count", "Teacher.count", "Classroom.count" ] do
+      post school_csv_import_url(@school), params: { file: csv_upload("students_missing_grade_level.csv") }
+    end
+
+    assert_redirected_to school_students_path(@school)
+    assert_match "Grade level can't be blank", flash[:alert]
+  end
+
+  test "should report a missing file" do
+    assert_no_difference "Student.count" do
+      post school_csv_import_url(@school)
+    end
+
+    assert_redirected_to school_students_path(@school)
+    assert_equal "Choose a CSV file to import.", flash[:alert]
+  end
+
+  test "should 404 for an unknown school" do
+    assert_no_difference "Student.count" do
+      post school_csv_import_url(school_id: 0), params: { file: csv_upload("students.csv") }
+    end
+
+    assert_response :not_found
+  end
+
+  private
+    def csv_upload(name)
+      fixture_file_upload(name, "text/csv")
+    end
+end

test/fixtures/files/students.csv

@@ -0,0 +1,4 @@
+Student First Name,Student Last Name,Grade Level,Class Name,Teacher,Teacher Email,Program,Program Level
+Alan,Turing,5,Room A,Nina Simone,nsimone@example.com,Know Your Health,basic
+Katherine,Johnson,5,Room A,Nina Simone,nsimone@example.com,Know Your Health,basic
+Mae,Jemison,6,Room B,Duke Ellington,dellington@example.com,3D Wellness,moderate