diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1799db44..93e8203a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Updated CI test matrix to support Rails 7.2, 8.0, 8.1 and Ruby 3.2, 3.3, 3.4, 4.0. ([#565](https://github.com/ice-cube-ruby/ice_cube/pull/565)) by [@nehresma](https://github.com/nehresma) and [@pacso](https://github.com/pacso)
 - Added backwards compatibility for Ruby 2.6+ by replacing anonymous block arguments with explicit block parameters and adding Rails/Ruby version compatibility exclusions in CI test matrix. ([#562](https://github.com/ice-cube-ruby/ice_cube/pull/562)) by [@scpike](https://github.com/scpike)
 
+### Fixed
+- Adding, checking and enumerating exception times no longer slows down quadratically with the number of exception times. ([#572](https://github.com/ice-cube-ruby/ice_cube/pull/572)) by [@xulkrnan](https://github.com/xulkrnan)
+
 ## [0.17.0] - 2024-07-18
 ### Added
 - Indonesian translations. ([#505](https://github.com/seejohnrun/ice_cube/pull/505)) by [@achmiral](https://github.com/achmiral)
diff --git a/lib/ice_cube/schedule.rb b/lib/ice_cube/schedule.rb
index 00fd2479..c823d46a 100644
--- a/lib/ice_cube/schedule.rb
+++ b/lib/ice_cube/schedule.rb
@@ -19,6 +19,7 @@ def initialize(start_time = nil, options = {})
       self.end_time = options[:end_time] if options[:end_time]
       @all_recurrence_rules = []
       @all_exception_rules = []
+      @exception_index = {}
       yield self if block_given?
     end
 
@@ -80,13 +81,22 @@ def remove_recurrence_rule(rule)
     # Add an exception rule to the schedule
     def add_exception_rule(rule)
       return if rule.nil?
-      @all_exception_rules << rule unless @all_exception_rules.include?(rule)
+      if rule.is_a?(SingleOccurrenceRule)
+        return if exception_index[:times].key?(rule.time.to_r)
+        exception_index[:times][rule.time.to_r] = true
+        exception_index[:seconds][rule.time.to_i] = true
+      else
+        return if @all_exception_rules.include?(rule)
+        exception_index[:rules] << rule
+      end
+      @all_exception_rules << rule
     end
     alias_method :exrule, :add_exception_rule
 
     # Remove an exception rule
     def remove_exception_rule(rule)
       res = @all_exception_rules.delete(rule)
+      rebuild_exception_index
       res.nil? ? [] : [res]
     end
 
@@ -136,6 +146,7 @@ def remove_exception_time(time)
       @all_exception_rules.delete_if do |rule|
         found = true if rule.is_a?(SingleOccurrenceRule) && rule.time == time
       end
+      rebuild_exception_index
       time if found
     end
     alias_method :remove_extime, :remove_exception_time
@@ -473,9 +484,27 @@ def full_required?
     # Return a boolean indicating whether or not a specific time
     # is excluded from the schedule
     def exception_time?(time)
-      @all_exception_rules.any? do |rule|
-        rule.on?(time, start_time)
-      end
+      exception_index[:seconds].key?(time.to_i) ||
+        exception_index[:rules].any? { |rule| rule.on?(time, start_time) }
+    end
+
+    # Exception times as exact values (for de-duplication) and whole seconds
+    # (for lookup), plus the remaining exception rules. Like the rule arrays,
+    # the index is shared with copies made by dup/clone, so it is only ever
+    # updated in place.
+    def exception_index
+      @exception_index ||= {}
+      rebuild_exception_index if @exception_index.empty?
+      @exception_index
+    end
+
+    def rebuild_exception_index
+      single, rules = @all_exception_rules.partition { |r| r.is_a?(SingleOccurrenceRule) }
+      (@exception_index ||= {}).replace(
+        times: single.to_h { |r| [r.time.to_r, true] },
+        seconds: single.to_h { |r| [r.time.to_i, true] },
+        rules: rules
+      )
     end
 
     def require_terminating_rules
diff --git a/spec/examples/schedule_spec.rb b/spec/examples/schedule_spec.rb
index 5b6052a5..c6edf3b7 100644
--- a/spec/examples/schedule_spec.rb
+++ b/spec/examples/schedule_spec.rb
@@ -701,6 +701,87 @@
       schedule = IceCube::Schedule.new Time.now
       expect(schedule.remove_exception_time(Time.now)).to be_nil
     end
+
+    it "should restore the occurrence at a removed exception time" do
+      schedule = IceCube::Schedule.new(t0 = Time.utc(2010, 1, 1))
+      schedule.rrule IceCube::Rule.hourly.count(3)
+      schedule.extime t0 + ONE_HOUR
+      expect(schedule.all_occurrences).to eq([t0, t0 + 2 * ONE_HOUR])
+      schedule.remove_exception_time t0 + ONE_HOUR
+      expect(schedule.all_occurrences).to eq([t0, t0 + ONE_HOUR, t0 + 2 * ONE_HOUR])
+    end
+
+    it "should allow an exception time to be added again after removal" do
+      schedule = IceCube::Schedule.new(t0 = Time.utc(2010, 1, 1))
+      schedule.rrule IceCube::Rule.hourly.count(3)
+      schedule.extime t0 + ONE_HOUR
+      schedule.remove_exception_time t0 + ONE_HOUR
+      schedule.extime t0 + ONE_HOUR
+      expect(schedule.exception_times).to eq([t0 + ONE_HOUR])
+      expect(schedule.all_occurrences).to eq([t0, t0 + 2 * ONE_HOUR])
+    end
+
+    it "should keep a copy and its original consistent with their exception times" do
+      schedule = IceCube::Schedule.new(t0 = Time.utc(2010, 1, 1))
+      schedule.rrule IceCube::Rule.hourly.count(4)
+      schedule.extime t0 + ONE_HOUR
+      schedule.all_occurrences
+      copy = schedule.dup
+      copy.remove_exception_time t0 + ONE_HOUR
+      copy.extime t0 + 2 * ONE_HOUR
+      [schedule, copy].each do |s|
+        expect(s.all_occurrences).to eq([0, 1, 2, 3].map { |h| t0 + h * ONE_HOUR } - s.exception_times)
+      end
+    end
+  end
+
+  describe :add_exception_time do
+    it "should not add the same exception time twice" do
+      schedule = IceCube::Schedule.new(t0 = Time.utc(2010, 1, 1))
+      2.times { schedule.extime t0 + ONE_HOUR }
+      expect(schedule.exception_times).to eq([t0 + ONE_HOUR])
+    end
+
+    it "should keep exception times that differ by a fraction of a second" do
+      schedule = IceCube::Schedule.new(t0 = Time.utc(2010, 1, 1))
+      schedule.extime t0
+      schedule.extime t0 + 0.5
+      expect(schedule.exception_times).to eq([t0, t0 + 0.5])
+    end
+
+    it "should skip an exception time given in another time zone" do
+      schedule = IceCube::Schedule.new(t0 = Time.utc(2010, 1, 1))
+      schedule.rrule IceCube::Rule.hourly.count(3)
+      schedule.extime((t0 + ONE_HOUR).getlocal("+09:00"))
+      expect(schedule.all_occurrences).to eq([t0, t0 + 2 * ONE_HOUR])
+    end
+
+    it "should combine exception times with exception rules" do
+      schedule = IceCube::Schedule.new(t0 = Time.utc(2010, 1, 1))
+      schedule.rrule IceCube::Rule.hourly.count(6)
+      schedule.extime t0 + ONE_HOUR
+      schedule.exrule IceCube::Rule.hourly(2)
+      expect(schedule.all_occurrences).to eq([t0 + 3 * ONE_HOUR, t0 + 5 * ONE_HOUR])
+    end
+
+    it "should handle thousands of exception times quickly" do
+      schedule = IceCube::Schedule.new(t0 = Time.utc(2010, 1, 1))
+      schedule.rrule IceCube::Rule.minutely(15).until(t0 + 32 * ONE_DAY)
+      (ONE_DAY...31 * ONE_DAY).step(15 * ONE_MINUTE) { |offset| schedule.extime t0 + offset }
+      expect(schedule.exception_times.size).to eq(2880)
+      expect(schedule.all_occurrences.size).to eq(2 * 96 + 1)
+    end
+  end
+
+  describe :remove_exception_rule do
+    it "should restore occurrences excluded by a removed exception rule" do
+      schedule = IceCube::Schedule.new(t0 = Time.utc(2010, 1, 1))
+      schedule.rrule IceCube::Rule.hourly.count(4)
+      schedule.extime t0 + ONE_HOUR
+      schedule.exrule IceCube::Rule.hourly(2)
+      schedule.remove_exception_rule IceCube::Rule.hourly(2)
+      expect(schedule.all_occurrences).to eq([t0, t0 + 2 * ONE_HOUR, t0 + 3 * ONE_HOUR])
+    end
   end
 
   describe :occurs_on? do