diff --git a/.env.example b/.env.example index 1d6a3ad..b7666c9 100644 --- a/.env.example +++ b/.env.example @@ -1 +1,5 @@ GEMINI_API_KEY= + +# Hosted defuddle worker (content extraction). URL defaults to the production worker. +DEFUDDLE_API_KEY= +DEFUDDLE_API_URL=https://deffudler.svc.canadasbuilding.com diff --git a/app/controllers/api/agent/commitment_matches_controller.rb b/app/controllers/api/agent/commitment_matches_controller.rb index 9be9502..625d656 100644 --- a/app/controllers/api/agent/commitment_matches_controller.rb +++ b/app/controllers/api/agent/commitment_matches_controller.rb @@ -8,10 +8,13 @@ def create matchable_id: params.require(:matchable_id), ) + # A match created by the agent has, by definition, already been reviewed. match.assign_attributes( relevance_score: params.require(:relevance_score), relevance_reasoning: params[:relevance_reasoning], matched_at: Time.current, + assessed: true, + assessed_at: Time.current, ) match.save! diff --git a/app/controllers/api/agent/pages_controller.rb b/app/controllers/api/agent/pages_controller.rb index 9fbc98c..4f3eb9e 100644 --- a/app/controllers/api/agent/pages_controller.rb +++ b/app/controllers/api/agent/pages_controller.rb @@ -1,22 +1,24 @@ module Api module Agent class PagesController < BaseController - ALLOWED_SUFFIXES = %w[.canada.ca .gc.ca].freeze + ALLOWED_SUFFIXES = %w[.canada.ca .gc.ca .parl.ca].freeze def fetch url = params.require(:url) government_id = params.require(:government_id) unless government_url?(url) - render json: { error: "URL must be on a *.canada.ca or *.gc.ca domain" }, status: :unprocessable_entity + render json: { error: "URL must be on a *.canada.ca, *.gc.ca, or *.parl.ca domain" }, status: :unprocessable_entity return end # Fetch and parse the page - response = HTTP.timeout(connect: 5, read: 20) - .headers("User-Agent" => "BuildCanada-Tracker/1.0") - .follow(max_hops: 3) - .get(url) + begin + response = PageFetcher.get(url) + rescue HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError, SocketError => e + render json: { error: "Failed to fetch: #{e.class.name.demodulize} (#{e.message})" }, status: :bad_gateway + return + end unless response.status.success? render json: { error: "Failed to fetch: HTTP #{response.status}" }, status: :bad_gateway @@ -31,8 +33,13 @@ def fetch end # Parse HTML to markdown - prepared_html = Defuddle.prepare_html(response.body.to_s) - parsed_markdown, _parsed_html = Defuddle.defuddle(prepared_html) + begin + prepared_html = Defuddle.prepare_html(response.body.to_s) + parsed_markdown, _parsed_html = Defuddle.defuddle(prepared_html, url: final_url) + rescue Defuddle::ParseError => e + render json: { error: "Failed to parse page: #{e.message}" }, status: :bad_gateway + return + end # Extract title and date from meta tags doc = Nokogiri::HTML(response.body.to_s) diff --git a/app/controllers/statcan_datasets_controller.rb b/app/controllers/statcan_datasets_controller.rb index 4634e42..a1a6d47 100644 --- a/app/controllers/statcan_datasets_controller.rb +++ b/app/controllers/statcan_datasets_controller.rb @@ -1,6 +1,11 @@ class StatcanDatasetsController < ApplicationController def show - dataset = StatcanDataset.find_by(name: params[:id]) - render json: dataset + dataset = StatcanDataset.find_by(name: params[:id]) || StatcanDataset.find_by(id: params[:id]) + + if dataset + render json: dataset + else + render json: { error: "Dataset not found" }, status: :not_found + end end end diff --git a/app/jobs/agent_evaluate_commitment_job.rb b/app/jobs/agent_evaluate_commitment_job.rb index afe265d..5d9c9c0 100644 --- a/app/jobs/agent_evaluate_commitment_job.rb +++ b/app/jobs/agent_evaluate_commitment_job.rb @@ -59,7 +59,7 @@ def build_cmd(prompt, hook_script:) "--system-prompt", system_prompt, "--allowedTools", allowed_tools.join(","), "--permission-mode", "bypassPermissions", - "--model", ENV.fetch("AGENT_MODEL", "claude-sonnet-4-6"), + "--model", ENV.fetch("AGENT_MODEL", "claude-sonnet-5"), "--output-format", "text", "--settings", hook_settings ] @@ -91,7 +91,7 @@ def agent_env(commitment_id: nil, entry_id: nil) "CLAUDE_CODE_OAUTH_TOKEN" => ENV["CLAUDE_CODE_OAUTH_TOKEN"], "RAILS_API_URL" => ENV.fetch("RAILS_API_URL", "http://localhost:3000"), "RAILS_API_KEY" => Rails.application.credentials.dig(:agent, :api_key) || ENV["AGENT_API_KEY"], - "AGENT_MODEL" => ENV.fetch("AGENT_MODEL", "claude-sonnet-4-6"), + "AGENT_MODEL" => ENV.fetch("AGENT_MODEL", "claude-sonnet-5"), "COMMITMENT_ID" => commitment_id&.to_s, "ENTRY_ID" => entry_id&.to_s, # Explicitly unset — subprocess must not access Rails credentials diff --git a/app/jobs/agent_process_entry_job.rb b/app/jobs/agent_process_entry_job.rb index 6ede719..86d0c86 100644 --- a/app/jobs/agent_process_entry_job.rb +++ b/app/jobs/agent_process_entry_job.rb @@ -48,7 +48,7 @@ def build_cmd(prompt, hook_script:) "--system-prompt", system_prompt, "--allowedTools", allowed_tools.join(","), "--permission-mode", "bypassPermissions", - "--model", ENV.fetch("AGENT_MODEL", "claude-opus-4-6"), + "--model", ENV.fetch("AGENT_MODEL", "claude-sonnet-5"), "--output-format", "text", "--settings", hook_settings ] @@ -79,7 +79,7 @@ def agent_env(commitment_id: nil, entry_id: nil) "CLAUDE_CODE_OAUTH_TOKEN" => ENV["CLAUDE_CODE_OAUTH_TOKEN"], "RAILS_API_URL" => ENV.fetch("RAILS_API_URL", "http://localhost:3000"), "RAILS_API_KEY" => Rails.application.credentials.dig(:agent, :api_key) || ENV["AGENT_API_KEY"], - "AGENT_MODEL" => ENV.fetch("AGENT_MODEL", "claude-opus-4-6"), + "AGENT_MODEL" => ENV.fetch("AGENT_MODEL", "claude-sonnet-5"), "COMMITMENT_ID" => commitment_id&.to_s, "ENTRY_ID" => entry_id&.to_s, # Explicitly unset — subprocess must not access Rails credentials diff --git a/app/jobs/commitment_assessment_cron_job.rb b/app/jobs/commitment_assessment_cron_job.rb deleted file mode 100644 index ba994b3..0000000 --- a/app/jobs/commitment_assessment_cron_job.rb +++ /dev/null @@ -1,12 +0,0 @@ -class CommitmentAssessmentCronJob < ApplicationJob - queue_as :default - - def perform - commitment_ids = CommitmentMatch.unassessed.high_relevance - .select(:commitment_id).distinct.pluck(:commitment_id) - - Commitment.where(id: commitment_ids).find_each do |commitment| - CommitmentAssessmentJob.perform_later(commitment) - end - end -end diff --git a/app/jobs/commitment_assessment_job.rb b/app/jobs/commitment_assessment_job.rb deleted file mode 100644 index 799e4ad..0000000 --- a/app/jobs/commitment_assessment_job.rb +++ /dev/null @@ -1,103 +0,0 @@ -class CommitmentAssessmentJob < ApplicationJob - queue_as :default - - def perform(commitment) - if commitment.criteria.empty? - commitment.generate_criteria!(inline: true) - end - - unassessed_matches = commitment.commitment_matches.unassessed.high_relevance.includes(:matchable) - return if unassessed_matches.empty? - - evidence_items = unassessed_matches.map(&:matchable).compact - latest_evidence_date = evidence_items.filter_map { |m| evidence_date_for(m) }.max || Time.current - - commitment.criteria.find_each do |criterion| - assessor = CriterionAssessor.create!(record: criterion) - assessor.extract!(assessor.prompt(criterion, evidence_items)) - - new_status = assessor.assessment["new_status"] - next if new_status == criterion.status - - evidence_index = assessor.assessment["primary_evidence_index"] - primary_evidence = evidence_items[evidence_index] if evidence_index&.between?(0, evidence_items.size - 1) - primary_evidence ||= evidence_items.first - source = source_for(primary_evidence, commitment.government) - - CriterionAssessment.create!( - criterion: criterion, - previous_status: criterion.status, - new_status: new_status, - evidence_notes: assessor.assessment["evidence_notes"], - assessed_at: latest_evidence_date, - source: source - ) - - criterion.update!( - status: new_status, - evidence_notes: assessor.assessment["evidence_notes"], - assessed_at: latest_evidence_date - ) - end - - unassessed_matches.update_all(assessed: true, assessed_at: latest_evidence_date) - commitment.update!(last_assessed_at: latest_evidence_date) - end - - private - - def evidence_date_for(matchable) - case matchable - when Entry then matchable.published_at - when Bill then [ matchable.passed_house_first_reading_at, matchable.latest_activity_at ].compact.max - when StatcanDataset then matchable.last_synced_at - end - end - - def source_for(matchable, government) - return nil unless matchable - - case matchable - when Bill - Source.find_or_create_by!( - government: government, - source_type: :other, - source_type_other: "Parliamentary Bill", - title: "#{matchable.bill_number_formatted} — #{matchable.short_title}", - url: "https://www.parl.ca/legisinfo/en/bill/#{matchable.parliament_number}-#{matchable.data&.dig('SessionNumber')}/#{matchable.bill_number_formatted}", - date: evidence_date_for(matchable)&.to_date - ) - when Entry - source_type, source_type_other = source_type_for_feed(matchable.feed) - attrs = { - government: government, - source_type: source_type, - title: matchable.title, - url: matchable.url, - date: matchable.published_at&.to_date - } - attrs[:source_type_other] = source_type_other if source_type_other - Source.find_or_create_by!(attrs) - when StatcanDataset - Source.find_or_create_by!( - government: government, - source_type: :other, - source_type_other: "Statistics Canada Dataset", - title: matchable.name, - url: matchable.statcan_url, - date: matchable.last_synced_at&.to_date - ) - end - end - - def source_type_for_feed(feed) - title = feed.title.to_s.downcase - if title.include?("gazette") - [ :gazette_notice, nil ] - elsif title.include?("committee") - [ :committee_report, nil ] - else - [ :other, feed.title ] - end - end -end diff --git a/app/models/chat.rb b/app/models/chat.rb index dba2bc4..1d320de 100644 --- a/app/models/chat.rb +++ b/app/models/chat.rb @@ -3,6 +3,16 @@ class Chat < ApplicationRecord belongs_to :record, polymorphic: true, optional: true + class << self + # Rows from retired pipelines (CriterionAssessor, CommitmentStatusDeriver) remain as an + # audit log after their classes were removed. Load them as plain Chats instead of raising. + def find_sti_class(type_name) + super + rescue ActiveRecord::SubclassNotFound + self + end + end + def system_prompt end diff --git a/app/models/commitment_status_deriver.rb b/app/models/commitment_status_deriver.rb deleted file mode 100644 index 44d57d9..0000000 --- a/app/models/commitment_status_deriver.rb +++ /dev/null @@ -1,120 +0,0 @@ -class CommitmentStatusDeriver < Chat - include Structify::Model - - MODEL = "gemini-3.1-flash-lite-preview" - - after_create { with_model(MODEL, provider: :gemini, assume_exists: true) } - - schema_definition do - version 1 - name "CommitmentStatusDeriver" - description "Derives overall commitment status from criteria assessments" - field :derivation, :object, properties: { - "recommended_status" => { type: "string", enum: %w[not_started in_progress completed abandoned] }, - "reasoning" => { type: "string", description: "Explanation of why this status was recommended" }, - "confidence" => { type: "number", description: "0.0 to 1.0" } - } - end - - def system_prompt - "You are a government accountability analyst determining the overall status of a commitment based on its evaluation criteria." - end - - def prompt(commitment) - criteria_summary = commitment.criteria.where.not(category: :success).map do |c| - status_label = c.status - notes = c.evidence_notes.present? ? "\n Evidence: #{c.evidence_notes.truncate(300)}" : "" - " [#{c.category}] #{c.description}\n Status: #{status_label}#{notes}" - end.join("\n\n") - - <<~PROMPT - Determine the overall status of this government commitment based on its evaluation criteria. - - COMMITMENT: - Title: #{commitment.title} - Description: #{commitment.description} - Type: #{commitment.commitment_type} - Current Status: #{commitment.status} - - MATCHED EVIDENCE SUMMARY: - #{format_evidence_summary(commitment)} - - CRITERIA AND THEIR CURRENT ASSESSMENTS: - #{criteria_summary.presence || "(No criteria assessed yet)"} - - STATUS DEFINITIONS: - - not_started: No evidence of any action taken on this commitment - - in_progress: Some action has been taken but the commitment is not yet fulfilled - - completed: The government has done what it said it would do — completion criteria are met - - abandoned: The government has explicitly reversed or dropped this commitment - - EVIDENCE HIERARCHY (strictly enforced): - For "completed" status, one of these is REQUIRED: - - Legislative commitments: A matched bill with Royal Assent, OR a Gazette Part III entry - - Regulatory commitments: A matched Gazette Part II entry (enacted regulation) - - Spending/program commitments: Departmental news showing the program is operational - - For "in_progress" status, one of these is REQUIRED: - - A matched bill that has been introduced and is progressing (but no Royal Assent yet) - - A Gazette Part I entry (proposed regulation) - - Appropriation voted (matched to appropriation bill) with program evidence - - Departmental news showing concrete implementation steps - - Budget announcements alone (Budget 2025 text, budget speeches) are NOT sufficient evidence - for "in_progress" or "completed". A budget promise without a bill, regulation, or program - launch is "not_started". - - RULES: - - If completion criteria are met AND strong evidence exists per the hierarchy, recommend "completed". - - Progress criteria being met with appropriate evidence suggests "in_progress". - - If criteria are mostly "not_assessed", keep status as "not_started" unless there's clear evidence of action. - - If the only evidence is budget text or platform promises, recommend "not_started". - - Never recommend "abandoned" unless there is explicit evidence of reversal. - PROMPT - end - - private - - def format_evidence_summary(commitment) - matches = commitment.commitment_matches.includes(:matchable) - return "(No matched evidence)" if matches.empty? - - summary_parts = [] - - bill_matches = matches.select { |m| m.matchable_type == "Bill" } - if bill_matches.any? - bill_matches.each do |m| - bill = m.matchable - next unless bill - - royal_assent = bill.received_royal_assent_at.present? ? "Royal Assent #{bill.received_royal_assent_at.to_date}" : "No Royal Assent" - house_stage = if bill.passed_house_third_reading_at - "Passed House 3R" - elsif bill.passed_house_second_reading_at - "Passed House 2R" - elsif bill.passed_house_first_reading_at - "Passed House 1R" - else - "Introduced" - end - summary_parts << "BILL: #{bill.bill_number_formatted} — #{bill.short_title} (#{house_stage}, #{royal_assent})" - end - end - - entry_matches = matches.select { |m| m.matchable_type == "Entry" } - if entry_matches.any? - entry_matches.group_by { |m| m.matchable&.feed }.each do |feed, feed_matches| - next unless feed - - summary_parts << "#{feed.title}: #{feed_matches.size} matched entries" - end - end - - statcan_matches = matches.select { |m| m.matchable_type == "StatcanDataset" } - if statcan_matches.any? - summary_parts << "StatCan datasets: #{statcan_matches.size} matched" - end - - summary_parts.presence&.join("\n") || "(No matched evidence)" - end -end diff --git a/app/models/criterion_assessor.rb b/app/models/criterion_assessor.rb deleted file mode 100644 index a5b0a99..0000000 --- a/app/models/criterion_assessor.rb +++ /dev/null @@ -1,85 +0,0 @@ -class CriterionAssessor < Chat - include Structify::Model - - MODEL = "gemini-3.1-pro-preview" - - after_create { with_model(MODEL, provider: :gemini, assume_exists: true) } - - schema_definition do - version 1 - name "CriterionAssessor" - description "Assesses a criterion against matched evidence" - field :assessment, :object, properties: { - "new_status" => { type: "string", enum: %w[not_assessed met not_met no_longer_applicable] }, - "evidence_notes" => { type: "string", description: "Explanation referencing specific evidence" }, - "confidence" => { type: "number", description: "0.0 to 1.0" }, - "primary_evidence_index" => { type: "integer", description: "0-based index of the most relevant evidence item used for this assessment, from the MATCHED EVIDENCE list" } - } - end - - def system_prompt - "You are a government accountability analyst assessing specific criteria against available evidence." - end - - def prompt(criterion, evidence_items) - <<~PROMPT - You are assessing a specific criterion for a government commitment based on available evidence. - - COMMITMENT: - Title: #{criterion.commitment.title} - Description: #{criterion.commitment.description} - Type: #{criterion.commitment.commitment_type} - - CRITERION TO ASSESS: - Category: #{criterion.category} - Description: #{criterion.description} - Verification Method: #{criterion.verification_method} - Current Status: #{criterion.status} - Previous Evidence: #{criterion.evidence_notes} - - MATCHED EVIDENCE: - #{format_evidence(evidence_items)} - - ASSESSMENT RULES: - - met: Clear evidence that this criterion is fully satisfied - - not_met: Criterion is not satisfied, even if there is some progress toward it - - not_assessed: Insufficient evidence to make any determination - - no_longer_applicable: The commitment has been abandoned or the criterion is moot - - There is NO "partially met" status. Criteria are binary: met or not_met. - Progress toward meeting a criterion does not make it met — that is tracked separately by progress criteria. - - EVIDENCE STANDARDS (critical): - - Budget announcements, budget speeches, and platform promises are NOT evidence of action. - They are statements of intent. A budget saying "we will do X" does not mean X is done or in progress. - - For COMPLETION criteria: require Royal Assent (for legislation), Gazette Part II publication - (for regulations), or operational program evidence (for spending/programs). - - For PROGRESS criteria: require a bill progressing through Parliament, a Gazette Part I - proposed regulation, or departmental news showing concrete implementation steps. - - A bill that has NOT received Royal Assent means the legislation is NOT enacted. - The Budget Implementation Act (Bill C-15) being introduced or progressing is evidence of - progress, NOT completion. - - Be CONSERVATIVE. Only mark as "met" if evidence clearly supports it. - If current status is already "met" and no contradictory evidence, keep it "met". - Reference specific evidence items in your evidence_notes. - Set primary_evidence_index to the 0-based index of the evidence item that most directly supports your assessment. - PROMPT - end - - private - - def format_evidence(items) - items.each_with_index.map do |item, idx| - case item - when Entry - "[#{idx}] ENTRY [#{item.feed&.title}]: #{item.title} (#{item.published_at&.to_date})\n#{item.parsed_markdown&.truncate(1000)}" - when Bill - royal_assent = item.received_royal_assent_at.present? ? "ENACTED (Royal Assent #{item.received_royal_assent_at.to_date})" : "NOT ENACTED" - "[#{idx}] BILL: #{item.bill_number_formatted} - #{item.short_title}\nStatus: #{royal_assent}\nLatest: #{item.latest_activity} (#{item.latest_activity_at&.to_date})\nHouse: 1R=#{item.passed_house_first_reading_at&.to_date} 2R=#{item.passed_house_second_reading_at&.to_date} 3R=#{item.passed_house_third_reading_at&.to_date}\nSenate: 1R=#{item.passed_senate_first_reading_at&.to_date} 2R=#{item.passed_senate_second_reading_at&.to_date} 3R=#{item.passed_senate_third_reading_at&.to_date}" - when StatcanDataset - "[#{idx}] STATCAN: #{item.name}\nData: #{item.current_data&.first(3)&.to_json}" - end - end.join("\n\n---\n\n") - end -end diff --git a/app/models/entry.rb b/app/models/entry.rb index 54acbb9..a4fa6eb 100644 --- a/app/models/entry.rb +++ b/app/models/entry.rb @@ -45,7 +45,7 @@ def fetch_data!(inline: false) end # Fetch data from external source - r = HTTP.timeout(connect: 10, read: 30).get(url) + r = PageFetcher.get(url, connect: 10, read: 30) if r.status >= 300 or r.status < 200 Rails.logger.error("Error fetching data for entry #{id}: #{r.status}") @@ -53,7 +53,7 @@ def fetch_data!(inline: false) end self.raw_html = Defuddle.prepare_html(r.body.to_s) - self.parsed_markdown, self.parsed_html = Defuddle.defuddle(raw_html) + self.parsed_markdown, self.parsed_html = Defuddle.defuddle(raw_html, url: url) self.scraped_at = Time.now self.is_index = document_relative_links.any? diff --git a/app/services/page_fetcher.rb b/app/services/page_fetcher.rb new file mode 100644 index 0000000..a3d1622 --- /dev/null +++ b/app/services/page_fetcher.rb @@ -0,0 +1,16 @@ +# Fetches external pages with a consistent crawler identity. +# +# canada.ca's edge rejects bare bot-style and spoofed-browser User-Agents (connection reset +# or indefinite hang) but accepts an honest "compatible" crawler string with a contact URL. +module PageFetcher + USER_AGENT = "Mozilla/5.0 (compatible; BuildCanadaTracker/1.0; +https://buildcanada.com)".freeze + + class << self + def get(url, connect: 5, read: 20, max_hops: 3) + HTTP.timeout(connect: connect, read: read) + .headers("User-Agent" => USER_AGENT) + .follow(max_hops: max_hops) + .get(url) + end + end +end diff --git a/app/views/avo/scraping_health/index.html.erb b/app/views/avo/scraping_health/index.html.erb index efc895b..ac3b61a 100644 --- a/app/views/avo/scraping_health/index.html.erb +++ b/app/views/avo/scraping_health/index.html.erb @@ -98,7 +98,11 @@ <%= entry.published_at ? time_ago_in_words(entry.published_at) + " ago" : "—" %> - <%= link_to entry.url, entry.url, target: "_blank", class: "text-blue-600 hover:underline", title: entry.url %> + <% if entry.url.to_s.match?(%r{\Ahttps?://}) %> + <%= link_to entry.url, entry.url, target: "_blank", class: "text-blue-600 hover:underline", title: entry.url %> + <% else %> + <%= entry.url %> + <% end %> <% end %> diff --git a/bin/brakeman b/bin/brakeman index ace1c9b..171ac12 100755 --- a/bin/brakeman +++ b/bin/brakeman @@ -2,6 +2,4 @@ require "rubygems" require "bundler/setup" -ARGV.unshift("--ensure-latest") - load Gem.bin_path("brakeman", "brakeman") diff --git a/config/brakeman.ignore b/config/brakeman.ignore new file mode 100644 index 0000000..ebfcdd0 --- /dev/null +++ b/config/brakeman.ignore @@ -0,0 +1,59 @@ +{ + "ignored_warnings": [ + { + "warning_type": "Cross-Site Scripting", + "warning_code": 4, + "fingerprint": "2aa3ad93f74ccdf22492d14b0d070e303e4a218aa07944e62fcc798e97d68a47", + "check_name": "LinkToHref", + "message": "Potentially unsafe model attribute in `link_to` href", + "file": "app/views/avo/scraping_health/index.html.erb", + "line": 102, + "link": "https://brakemanscanner.org/docs/warning_types/link_to_href", + "code": "link_to((Unresolved Model).new.url, (Unresolved Model).new.url, :target => \"_blank\", :class => \"text-blue-600 hover:underline\", :title => (Unresolved Model).new.url)", + "render_path": [ + { + "type": "controller", + "class": "Avo::ScrapingHealthController", + "method": "index", + "line": 23, + "file": "app/controllers/avo/scraping_health_controller.rb", + "rendered": { + "name": "avo/scraping_health/index", + "file": "app/views/avo/scraping_health/index.html.erb" + } + } + ], + "location": { + "type": "template", + "template": "avo/scraping_health/index" + }, + "user_input": "(Unresolved Model).new.url", + "confidence": "Weak", + "cwe_id": [ + 79 + ], + "note": "entry.url is rendered as a link only when it starts with http:// or https:// (see the guard in the view); Brakeman cannot see the conditional." + }, + { + "warning_type": "Unmaintained Dependency", + "warning_code": 122, + "fingerprint": "98b26f60d776fd41ee6f088c833725145be9aac2d7c5b33780241c273622db42", + "check_name": "EOLRails", + "message": "Support for Rails 8.0.2 ends on 2026-10-07", + "file": "Gemfile.lock", + "line": 331, + "link": "https://brakemanscanner.org/docs/warning_types/unmaintained_dependency/", + "code": null, + "render_path": null, + "location": null, + "user_input": null, + "confidence": "Medium", + "cwe_id": [ + 1104 + ], + "note": "Rails 8.0.x support ends 2026-10-07; the Rails upgrade is tracked separately from this scan." + } + ], + "updated": "2026-09-10", + "brakeman_version": "8.0.4" +} diff --git a/config/initializers/good_job.rb b/config/initializers/good_job.rb index 33fd919..d295a0c 100644 --- a/config/initializers/good_job.rb +++ b/config/initializers/good_job.rb @@ -35,12 +35,6 @@ description: "Generates evaluation criteria for commitments that don't have them yet", enabled_by_default: -> { Rails.env.production? } }, - commitment_assessment: { - cron: "0 */6 * * *", # Every 6 hours - class: "CommitmentAssessmentCronJob", - description: "Assess commitments with new evidence matches", - enabled_by_default: -> { Rails.env.production? } - }, target_date_extraction: { cron: "30 4 * * *", # Daily at 4:30 AM class: "TargetDateExtractionCronJob", diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 578e5e0..f3246bb 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -15,6 +15,8 @@ services: - SMTP_PASSWORD - SMTP_ADDRESS - GEMINI_API_KEY + - DEFUDDLE_API_KEY + - DEFUDDLE_API_URL - APPSIGNAL_PUSH_API_KEY - CLAUDE_CODE_OAUTH_TOKEN depends_on: @@ -42,6 +44,8 @@ services: - SMTP_PASSWORD - SMTP_ADDRESS - GEMINI_API_KEY + - DEFUDDLE_API_KEY + - DEFUDDLE_API_URL - APPSIGNAL_PUSH_API_KEY - CLAUDE_CODE_OAUTH_TOKEN restart: unless-stopped diff --git a/lib/defuddle.rb b/lib/defuddle.rb index d70fb93..6e03aa0 100644 --- a/lib/defuddle.rb +++ b/lib/defuddle.rb @@ -1,24 +1,45 @@ +require "http" +require "json" + +# Client for the hosted BuildCanada defuddle worker, which extracts the main content of a +# web page and returns it as markdown plus a cleaned HTML fragment. +# +# Configure with DEFUDDLE_API_KEY (required) and DEFUDDLE_API_URL (optional). module Defuddle - def self.defuddle(html) - temp_file = Tempfile.new("entry_html", encoding: "utf-8") - temp_file.write(html) + class ParseError < StandardError; end - md_json, err, status = Open3.capture3("defuddle", "parse", temp_file.path, "-m", "-j") - md_html, err, status = Open3.capture3("defuddle", "parse", temp_file.path, "-j") + DEFAULT_API_URL = "https://deffudler.svc.canadasbuilding.com".freeze - # replace anything before the first { deffudle returns errors and is dumb here. - md_json = "{" + md_json.split("{", 2).last - html_json = "{" + md_html.split("{", 2).last + class << self + # Sends already-fetched HTML to the worker and returns [markdown_content, html_content]. + # The page URL is passed along so relative links and site-specific rules resolve correctly. + def defuddle(html, url:) + response = HTTP.timeout(connect: 5, read: 60) + .headers("X-API-Key" => api_key, "Content-Type" => "application/json") + .post("#{api_url}/api/convert", json: { url: url, html: html }) - return JSON.parse(md_json)["content"], JSON.parse(html_json)["content"] - ensure - temp_file.close - temp_file.unlink - end + body = JSON.parse(response.body.to_s) + raise ParseError, "defuddle service returned HTTP #{response.status}: #{body["error"]}" unless response.status.success? + + [ body["content"], body["html"] ] + rescue HTTP::Error, JSON::ParserError => e + raise ParseError, "defuddle service request failed: #{e.message}" + end + + def prepare_html(html) + ic = Iconv.new("UTF-8//IGNORE", "UTF-8") + + ic.iconv(html + " ")[0..-2] + end + + private - def self.prepare_html(html) - ic = Iconv.new("UTF-8//IGNORE", "UTF-8") + def api_url + ENV.fetch("DEFUDDLE_API_URL", DEFAULT_API_URL).chomp("/") + end - ic.iconv(html + " ")[0..-2] + def api_key + ENV["DEFUDDLE_API_KEY"].presence || raise(ParseError, "DEFUDDLE_API_KEY is not set") + end end end diff --git a/lib/tasks/commitment_backfill.rake b/lib/tasks/commitment_backfill.rake index bade9a7..95c674a 100644 --- a/lib/tasks/commitment_backfill.rake +++ b/lib/tasks/commitment_backfill.rake @@ -42,19 +42,6 @@ namespace :commitments do puts "Enqueued relevance filtering for #{count} StatCan datasets" end - desc "Phase 5: Run initial assessment on all commitments with unassessed matches" - task assess: :environment do - count = 0 - commitment_ids = CommitmentMatch.unassessed.high_relevance - .select(:commitment_id).distinct.pluck(:commitment_id) - - Commitment.where(id: commitment_ids).find_each do |commitment| - CommitmentAssessmentJob.perform_later(commitment) - count += 1 - end - puts "Enqueued assessment for #{count} commitments" - end - desc "Run all backfill phases in sequence" task all: :environment do puts "=== Phase 1: Generate Criteria ===" @@ -64,8 +51,6 @@ namespace :commitments do puts " rake commitments:backfill:entries" puts " rake commitments:backfill:bills" puts " rake commitments:backfill:statcan" - puts "Then after those finish:" - puts " rake commitments:backfill:assess" end end end diff --git a/lib/tasks/matching_gaps.rake b/lib/tasks/matching_gaps.rake index 21a039d..747df8a 100644 --- a/lib/tasks/matching_gaps.rake +++ b/lib/tasks/matching_gaps.rake @@ -274,7 +274,6 @@ namespace :matching do puts "\n" + "=" * 70 puts "All gap-filling jobs enqueued." - puts "After jobs complete, run: rake matching:reassess" puts "=" * 70 end diff --git a/lib/tasks/reevaluate.rake b/lib/tasks/reevaluate.rake deleted file mode 100644 index c35696f..0000000 --- a/lib/tasks/reevaluate.rake +++ /dev/null @@ -1,117 +0,0 @@ -namespace :matching do - desc "Re-assess all commitments that have unassessed high-relevance matches" - task reassess: :environment do - commitment_ids = CommitmentMatch.unassessed.high_relevance - .select(:commitment_id).distinct.pluck(:commitment_id) - - puts "Found #{commitment_ids.size} commitments with unassessed high-relevance matches" - - count = 0 - Commitment.where(id: commitment_ids).find_each do |commitment| - CommitmentAssessmentJob.perform_later(commitment) - count += 1 - end - - puts "Enqueued assessment for #{count} commitments" - end - - desc "Report on commitment statuses and evidence quality" - task status_report: :environment do - puts "=" * 70 - puts "COMMITMENT STATUS REPORT" - puts "=" * 70 - - total = Commitment.count - by_status = Commitment.group(:status).count - puts "\nBy Status:" - by_status.each do |status, count| - pct = (count.to_f / total * 100).round(1) - puts " #{status}: #{count} (#{pct}%)" - end - - puts "\nEvidence Quality:" - - # Commitments with bill matches - with_bill = CommitmentMatch.where(matchable_type: "Bill") - .distinct.count(:commitment_id) - puts " With bill match: #{with_bill}" - - # With Royal Assent bills - royal_assent_bill_ids = Bill.where.not(received_royal_assent_at: nil).pluck(:id) - with_royal_assent = CommitmentMatch.where(matchable_type: "Bill", matchable_id: royal_assent_bill_ids) - .distinct.count(:commitment_id) - puts " With Royal Assent bill: #{with_royal_assent}" - - # With Gazette entries - gazette_ii_feed = Feed.find_by("title ILIKE ?", "%Gazette Part II%") - gazette_iii_feed = Feed.find_by("title ILIKE ?", "%Gazette Part III%") - - if gazette_ii_feed - g2_entry_ids = gazette_ii_feed.entries.pluck(:id) - with_g2 = CommitmentMatch.where(matchable_type: "Entry", matchable_id: g2_entry_ids) - .distinct.count(:commitment_id) - puts " With Gazette Part II match: #{with_g2}" - end - - if gazette_iii_feed - g3_entry_ids = gazette_iii_feed.entries.pluck(:id) - with_g3 = CommitmentMatch.where(matchable_type: "Entry", matchable_id: g3_entry_ids) - .distinct.count(:commitment_id) - puts " With Gazette Part III match: #{with_g3}" - end - - # With departmental news - dept_feeds = Feed.where("title ILIKE ANY(ARRAY[?])", [ - "%News Releases%", "%Press Releases%" - ]) - dept_entry_ids = Entry.where(feed: dept_feeds).pluck(:id) - with_dept = CommitmentMatch.where(matchable_type: "Entry", matchable_id: dept_entry_ids) - .distinct.count(:commitment_id) - puts " With departmental news match: #{with_dept}" - - # Criteria assessment coverage - puts "\nCriteria Assessment:" - total_criteria = Criterion.count - assessed = Criterion.where.not(status: :not_assessed).count - puts " Total criteria: #{total_criteria}" - puts " Assessed: #{assessed} (#{(assessed.to_f / total_criteria * 100).round(1)}%)" - puts " Unassessed: #{total_criteria - assessed}" - - # Evidence hierarchy violations - puts "\nEvidence Hierarchy Check:" - completed = Commitment.completed - completed.find_each do |c| - bill_matches = c.commitment_matches.where(matchable_type: "Bill") - entry_matches = c.commitment_matches.where(matchable_type: "Entry") - - has_royal_assent = bill_matches.any? do |m| - m.matchable.received_royal_assent_at.present? - end - - gazette_entry_ids = [] - gazette_entry_ids += g2_entry_ids if gazette_ii_feed - gazette_entry_ids += g3_entry_ids if gazette_iii_feed - - has_gazette = entry_matches.where(matchable_id: gazette_entry_ids).exists? if gazette_entry_ids.any? - - has_dept_news = entry_matches.where(matchable_id: dept_entry_ids).exists? if dept_entry_ids.any? - - unless has_royal_assent || has_gazette || has_dept_news - puts " WARNING: Commitment ##{c.id} '#{c.title.truncate(60)}' marked completed without strong evidence" - end - end - - puts "=" * 70 - end - - desc "Full pipeline: fix gaps, reassess, rederive (run sequentially as jobs complete)" - task full_pipeline: :environment do - puts "Step 1: Fixing matching gaps..." - Rake::Task["matching:all_gaps"].invoke - - puts "\nJobs are enqueued. The full pipeline is:" - puts " 1. rake matching:all_gaps (done — jobs running)" - puts " 2. rake matching:reassess (run after gap jobs finish)" - puts " 3. rake matching:status_report (run after assessment jobs finish)" - end -end diff --git a/test/jobs/commitment_relevance_filter_job_test.rb b/test/jobs/commitment_relevance_filter_job_test.rb index 06dfff0..fb1713e 100644 --- a/test/jobs/commitment_relevance_filter_job_test.rb +++ b/test/jobs/commitment_relevance_filter_job_test.rb @@ -108,13 +108,13 @@ class CommitmentRelevanceFilterJobTest < ActiveJob::TestCase assert_not_includes result.to_a, commitment end - test "excludes abandoned commitments regardless of date" do + test "excludes broken commitments regardless of date" do commitment = Commitment.create!( government: @government, - title: "Abandoned commitment", - description: "This was abandoned", + title: "Broken commitment", + description: "This was broken", commitment_type: :spending, - status: :abandoned, + status: :broken, date_promised: Date.new(2025, 1, 1) )