RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Serialize dates and times as ISO 8601 in ScrapyJSONEncoder (#7918)

This commit is contained in:
Adrian 2026-08-10 11:07:42 +02:00 committed by GitHub
parent 4cc2356637
commit fe96c1f54b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 15 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/topics/extensions.rst
View File

@ -374,8 +374,8 @@ This extension periodically logs rich stat data as a JSON object::
"elapsed": 360.008903,
"log_interval": 60.0,
"log_interval_real": 60.006694,
"start_time": "2023-08-03 23:24:57",
"utcnow": "2023-08-03 23:30:57"
"start_time": "2023-08-03T23:24:57.148903+00:00",
"utcnow": "2023-08-03T23:30:57.157806+00:00"
}
}

11
scrapy/utils/serialize.py
View File

@ -10,18 +10,11 @@ from scrapy.http import Request, Response
class ScrapyJSONEncoder(json.JSONEncoder):
DATE_FORMAT = "%Y-%m-%d"
TIME_FORMAT = "%H:%M:%S"
def default(self, o: Any) -> Any:
if isinstance(o, set):
return list(o)
if isinstance(o, datetime.datetime):
return o.strftime(f"{self.DATE_FORMAT} {self.TIME_FORMAT}")
if isinstance(o, datetime.date):
return o.strftime(self.DATE_FORMAT)
if isinstance(o, datetime.time):
return o.strftime(self.TIME_FORMAT)
if isinstance(o, (datetime.datetime, datetime.date, datetime.time)):
return o.isoformat()
if isinstance(o, decimal.Decimal):
return str(o)
if isinstance(o, defer.Deferred):

4
tests/test_exporters.py
View File

@ -578,7 +578,7 @@ class TestJsonLinesItemExporter(TestBaseItemExporter):
self.ie.finish_exporting()
del self.ie # See the first “del self.ie” in this file for context.
exported = json.loads(to_unicode(self.output.getvalue()))
item["time"] = str(item["time"])
item["time"] = item["time"].isoformat()
assert exported == item
@ -661,7 +661,7 @@ class TestJsonItemExporter(TestJsonLinesItemExporter):
self.ie.finish_exporting()
del self.ie # See the first “del self.ie” in this file for context.
exported = json.loads(to_unicode(self.output.getvalue()))
item["time"] = str(item["time"])
item["time"] = item["time"].isoformat()
assert exported == [item]

10
tests/test_utils_serialize.py
View File

@ -20,11 +20,17 @@ class TestJsonEncoder:
def test_encode_decode(self, encoder: ScrapyJSONEncoder) -> None:
dt = datetime.datetime(2010, 1, 2, 10, 11, 12)
dts = "2010-01-02 10:11:12"
dts = "2010-01-02T10:11:12"
dt_aware = datetime.datetime(
2010, 1, 2, 10, 11, 12, 133700, tzinfo=datetime.timezone.utc
)
dt_awares = "2010-01-02T10:11:12.133700+00:00"
d = datetime.date(2010, 1, 2)
ds = "2010-01-02"
t = datetime.time(10, 11, 12)
ts = "10:11:12"
t_us = datetime.time(10, 11, 12, 133700)
t_uss = "10:11:12.133700"
dec = Decimal("1000.12")
decs = "1000.12"
s = {"foo"}
@ -36,7 +42,9 @@ class TestJsonEncoder:
("foo", "foo"),
(d, ds),
(t, ts),
(t_us, t_uss),
(dt, dts),
(dt_aware, dt_awares),
(dec, decs),
(["foo", d], ["foo", ds]),
(s, ss),