Merge pull request #514 from seperman/dev

8.2.0

.github/workflows/main.yaml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         architecture: ["x64"]
     steps:
     - uses: actions/checkout@v2
@@ -23,7 +23,7 @@ jobs:
         architecture: ${{ matrix.architecture }}
     - name: Cache pip 3.8
       if: matrix.python-version == 3.8
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         # This path is specific to Ubuntu
         path: ~/.cache/pip
@@ -34,7 +34,9 @@ jobs:
           ${{ runner.os }}-
     - name: Cache pip
       if: matrix.python-version != 3.8
-      uses: actions/cache@v2
+      env:
+        PYO3_USE_ABI3_FORWARD_COMPATIBILITY: "1"
+      uses: actions/cache@v4
       with:
         # This path is specific to Ubuntu
         path: ~/.cache/pip
@@ -44,9 +46,9 @@ jobs:
           ${{ runner.os }}-pip-
           ${{ runner.os }}-
     - name: Upgrade setuptools
-      if: matrix.python-version => 3.12
+      if: matrix.python-version >= 3.12
       run: |
-        # workaround for 3.13, SEE: https://github.com/pypa/setuptools/issues/3661#issuecomment-1813845177
+        # workaround for 3.12, SEE: https://github.com/pypa/setuptools/issues/3661#issuecomment-1813845177
         pip install --upgrade setuptools
     - name: Install dependencies
       if: matrix.python-version > 3.9
@@ -55,23 +57,23 @@ jobs:
       if: matrix.python-version <= 3.9
       run: pip install -r requirements-dev3.8.txt
     - name: Lint with flake8
-      if: matrix.python-version == 3.13
+      if: matrix.python-version == 3.12
       run: |
         # stop the build if there are Python syntax errors or undefined names
         flake8 deepdiff --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 deepdiff --count --exit-zero --max-complexity=26 --max-line-lengt=250 --statistics
     - name: Test with pytest and get the coverage
-      if: matrix.python-version == 3.13
+      if: matrix.python-version == 3.12
       run: |
         pytest --benchmark-disable --cov-report=xml --cov=deepdiff tests/ --runslow
     - name: Test with pytest and no coverage report
-      if: matrix.python-version != 3.13
+      if: matrix.python-version != 3.12
       run: |
         pytest --benchmark-disable
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v4
-      if: matrix.python-version == 3.13
+      if: matrix.python-version == 3.12
       env:
         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
       with:

CHANGELOG.md

@@ -1,5 +1,9 @@
 # DeepDiff Change log
 
+- v8-2-0
+    - Small optimizations so we don't load functions that are not needed
+    - Updated the minimum version of Orderly-set 
+    - Normalize all datetimes into UTC. Assume timezone naive datetimes are UTC. 
 
 - v8-1-0
     - Removing deprecated lines from setup.py

README.md

@@ -23,6 +23,12 @@ Tested on Python 3.8+ and PyPy3.
 
 Please check the [ChangeLog](CHANGELOG.md) file for the detailed information.
 
+DeepDiff 8-2-0
+
+- Small optimizations so we don't load functions that are not needed
+- Updated the minimum version of Orderly-set 
+- Normalize all datetimes into UTC. Assume timezone naive datetimes are UTC. 
+
 DeepDiff 8-1-0
 
 - Removing deprecated lines from setup.py
@@ -40,56 +46,6 @@ DeepDiff 8-1-0
 - Fixes accessing the affected_root_keys property on the diff object returned by DeepDiff fails when one of the dicts is empty
 - Fixes accessing the affected_root_keys property on the diff object returned by DeepDiff fails when one of the dicts is empty #508
 
-DeepDiff 8-0-1
-
-- Bugfix. Numpy should be optional.
-
-DeepDiff 8-0-0
-
-With the introduction of `threshold_to_diff_deeper`, the values returned are different than in previous versions of DeepDiff. You can still get the older values by setting `threshold_to_diff_deeper=0`. However to signify that enough has changed in this release that the users need to update the parameters passed to DeepDiff, we will be doing a major version update.
-
-- `use_enum_value=True` makes it so when diffing enum, we use the enum's value. It makes it so comparing an enum to a string or any other value is not reported as a type change.
-- `threshold_to_diff_deeper=float` is a number between 0 and 1. When comparing dictionaries that have a small intersection of keys, we will report the dictionary as a `new_value` instead of reporting individual keys changed. If you set it to zero, you get the same results as DeepDiff 7.0.1 and earlier, which means this feature is disabled. The new default is 0.33 which means if less that one third of keys between dictionaries intersect, report it as a new object.
-- Deprecated `ordered-set` and switched to `orderly-set`. The `ordered-set` package was not being maintained anymore and starting Python 3.6, there were better options for sets that ordered. I forked one of the new implementations, modified it, and published it as `orderly-set`.
-- Added `use_log_scale:bool` and `log_scale_similarity_threshold:float`. They can be used to ignore small changes in numbers by comparing their differences in logarithmic space. This is different than ignoring the difference based on significant digits.
-- json serialization of reversed lists.
-- Fix for iterable moved items when `iterable_compare_func` is used.
-- Pandas and Polars support.
-
-DeepDiff 7-0-1
-
-- Fixes the translation between Difflib opcodes and Delta flat rows.
-
-DeepDiff 7-0-0
-
-- DeepDiff 7 comes with an improved delta object. [Delta to flat dictionaries](https://zepworks.com/deepdiff/current/serialization.html#delta-serialize-to-flat-dictionaries) have undergone a major change. We have also introduced [Delta serialize to flat rows](https://zepworks.com/deepdiff/current/serialization.html#delta-serialize-to-flat-rows).
-- Subtracting delta objects have dramatically improved at the cost of holding more metadata about the original objects.
-- When `verbose=2`, and the "path" of an item has changed in a report between t1 and t2, we include it as `new_path`.
-- `path(use_t2=True)` returns the correct path to t2 in any reported change in the [`tree view`](https://zepworks.com/deepdiff/current/view.html#tree-view)
-- Python 3.7 support is dropped and Python 3.12 is officially supported.
-
-
-DeepDiff 6-7-1
-
-- Support for subtracting delta objects when iterable_compare_func is used.
-- Better handling of force adding a delta to an object. 
-- Fix for [`Can't compare dicts with both single and double quotes in keys`](https://github.com/seperman/deepdiff/issues/430)
-- Updated docs for Inconsistent Behavior with math_epsilon and ignore_order = True
-
-DeepDiff 6-7-0
-
-- Delta can be subtracted from other objects now.
-- verify_symmetry is deprecated. Use bidirectional instead.
-- always_include_values flag in Delta can be enabled to include values in the delta for every change.
-- Fix for Delta.__add__ breaks with esoteric dict keys.
-- You can load a delta from the list of flat dictionaries.
-
-DeepDiff 6-6-1
-
-- Fix for [DeepDiff raises decimal exception when using significant digits](https://github.com/seperman/deepdiff/issues/426)
-- Introducing group_by_sort_key
-- Adding group_by 2D. For example `group_by=['last_name', 'zip_code']`
-
 
 ## Installation
 

deepdiff/diff.py

@@ -8,14 +8,16 @@
 import difflib
 import logging
 import types
+import datetime
 from enum import Enum
 from copy import deepcopy
 from math import isclose as is_close
-from typing import List, Dict, IO, Callable, Set, Union, Any, Pattern, Tuple, Optional
+from typing import List, Dict, Callable, Union, Any, Pattern, Tuple, Optional
 from collections.abc import Mapping, Iterable, Sequence
 from collections import defaultdict
 from inspect import getmembers
 from itertools import zip_longest
+from functools import lru_cache
 from deepdiff.helper import (strings, bytes_type, numbers, uuids, datetimes, ListItemRemovedOrAdded, notpresent,
                              IndexedHash, unprocessed, add_to_frozen_set, basic_types,
                              convert_item_or_items_into_set_else_none, get_type,
@@ -1123,6 +1125,7 @@ def _create_hashtable(self, level, t):
         return local_hashes
 
     @staticmethod
+    @lru_cache(maxsize=2028)
     def _get_distance_cache_key(added_hash, removed_hash):
         key1, key2 = (added_hash, removed_hash) if added_hash > removed_hash else (removed_hash, added_hash)
         if isinstance(key1, int):
@@ -1485,7 +1488,15 @@ def _diff_numbers(self, level, local_tree=None, report_type_change=True):
             if t1_s != t2_s:
                 self._report_result('values_changed', level, local_tree=local_tree)
 
-    def _diff_datetimes(self, level, local_tree=None):
+    def _diff_datetime(self, level, local_tree=None):
+        """Diff DateTimes"""
+        level.t1 = datetime_normalize(self.truncate_datetime, level.t1)
+        level.t2 = datetime_normalize(self.truncate_datetime, level.t2)
+
+        if level.t1 != level.t2:
+            self._report_result('values_changed', level, local_tree=local_tree)
+
+    def _diff_time(self, level, local_tree=None):
         """Diff DateTimes"""
         if self.truncate_datetime:
             level.t1 = datetime_normalize(self.truncate_datetime, level.t1)
@@ -1668,8 +1679,11 @@ def _diff(self, level, parents_ids=frozenset(), _original_type=None, local_tree=
         elif isinstance(level.t1, strings):
             self._diff_str(level, local_tree=local_tree)
 
-        elif isinstance(level.t1, datetimes):
-            self._diff_datetimes(level, local_tree=local_tree)
+        elif isinstance(level.t1, datetime.datetime):
+            self._diff_datetime(level, local_tree=local_tree)
+
+        elif isinstance(level.t1, (datetime.date, datetime.timedelta, datetime.time)):
+            self._diff_time(level, local_tree=local_tree)
 
         elif isinstance(level.t1, uuids):
             self._diff_uuids(level, local_tree=local_tree)

deepdiff/helper.py

@@ -1,7 +1,6 @@
 import sys
 import re
 import os
-import math
 import datetime
 import uuid
 import logging
@@ -624,12 +623,29 @@ def datetime_normalize(truncate_datetime, obj):
         elif truncate_datetime == 'day':
             obj = obj.replace(hour=0, minute=0, second=0, microsecond=0)
     if isinstance(obj, datetime.datetime):
-        obj = obj.replace(tzinfo=datetime.timezone.utc)
+        if has_timezone(obj):
+            obj = obj.astimezone(datetime.timezone.utc)
+        else:
+            obj = obj.replace(tzinfo=datetime.timezone.utc)
     elif isinstance(obj, datetime.time):
         obj = time_to_seconds(obj)
     return obj
 
 
+def has_timezone(dt):
+    """
+    Function to check if a datetime object has a timezone
+
+    Checking dt.tzinfo.utcoffset(dt) ensures that the datetime object is truly timezone-aware
+    because some datetime objects may have a tzinfo attribute that is not None but still
+    doesn't provide a valid offset.
+
+    Certain tzinfo objects, such as pytz.timezone(None), can exist but do not provide meaningful UTC offset information.
+    If tzinfo is present but calling .utcoffset(dt) returns None, the datetime is not truly timezone-aware.
+    """
+    return dt.tzinfo is not None and dt.tzinfo.utcoffset(dt) is not None
+
+
 def get_truncate_datetime(truncate_datetime):
     """
     Validates truncate_datetime value