improve comments index mapping, serializer
This commit is contained in:
Simon
parent
36ab4fb668
commit
50efe1650a
|
|
@ -674,37 +674,11 @@
|
|||
{
|
||||
"index_name": "comment",
|
||||
"expected_map": {
|
||||
"youtube_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"comment_channel_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_comments": {
|
||||
"properties": {
|
||||
"comment_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_text": {
|
||||
"type": "text"
|
||||
},
|
||||
"comment_timestamp": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"comment_time_text": {
|
||||
"type": "text"
|
||||
},
|
||||
"comment_likecount": {
|
||||
"type": "long"
|
||||
},
|
||||
"comment_is_favorited": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"comment_author": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
|
|
@ -718,16 +692,42 @@
|
|||
"comment_author_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_author_thumbnail": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_author_is_uploader": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"comment_author_thumbnail": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_is_favorited": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"comment_likecount": {
|
||||
"type": "long"
|
||||
},
|
||||
"comment_parent": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"comment_text": {
|
||||
"type": "text"
|
||||
},
|
||||
"comment_time_text": {
|
||||
"type": "text"
|
||||
},
|
||||
"comment_timestamp": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
}
|
||||
}
|
||||
},
|
||||
"comment_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
},
|
||||
"youtube_id": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"expected_set": {
|
||||
|
|
|
|||
|
|
@ -147,17 +147,18 @@ class VideoListQuerySerializer(serializers.Serializer):
|
|||
class CommentItemSerializer(serializers.Serializer):
|
||||
"""serialize comment item"""
|
||||
|
||||
comment_id = serializers.CharField()
|
||||
comment_text = serializers.CharField()
|
||||
comment_timestamp = serializers.IntegerField()
|
||||
comment_time_text = serializers.CharField()
|
||||
comment_likecount = serializers.IntegerField()
|
||||
comment_is_favorited = serializers.BooleanField()
|
||||
comment_author = serializers.CharField()
|
||||
comment_author_id = serializers.CharField()
|
||||
comment_author_thumbnail = serializers.URLField()
|
||||
comment_author_is_uploader = serializers.BooleanField()
|
||||
comment_author_thumbnail = serializers.URLField()
|
||||
comment_id = serializers.CharField()
|
||||
comment_is_favorited = serializers.BooleanField()
|
||||
comment_likecount = serializers.IntegerField()
|
||||
comment_parent = serializers.CharField()
|
||||
comment_text = serializers.CharField()
|
||||
comment_time_text = serializers.CharField()
|
||||
comment_timestamp = serializers.IntegerField()
|
||||
|
||||
comment_replies = serializers.SerializerMethodField()
|
||||
|
||||
@extend_schema_field(serializers.ListField())
|
||||
|
|
@ -171,10 +172,10 @@ class CommentItemSerializer(serializers.Serializer):
|
|||
class CommentsSerializer(serializers.Serializer):
|
||||
"""serialize comments as indexed"""
|
||||
|
||||
youtube_id = serializers.CharField()
|
||||
comment_last_refresh = serializers.IntegerField()
|
||||
comment_channel_id = serializers.CharField()
|
||||
comment_comments = CommentItemSerializer(many=True)
|
||||
comment_last_refresh = serializers.IntegerField()
|
||||
youtube_id = serializers.CharField()
|
||||
|
||||
|
||||
class PlaylistNavMetaSerializer(serializers.Serializer):
|
||||
|
|
|
|||
|
|
@ -39,10 +39,10 @@ class Comments:
|
|||
self.format_comments(comments_raw)
|
||||
|
||||
self.json_data = {
|
||||
"youtube_id": self.youtube_id,
|
||||
"comment_last_refresh": int(datetime.now().timestamp()),
|
||||
"comment_channel_id": channel_id,
|
||||
"comment_comments": self.comments_format,
|
||||
"comment_last_refresh": int(datetime.now().timestamp()),
|
||||
"youtube_id": self.youtube_id,
|
||||
}
|
||||
if upload:
|
||||
self.upload_comments()
|
||||
|
|
@ -124,20 +124,20 @@ class Comments:
|
|||
if not comment.get("author"):
|
||||
comment["author"] = comment.get("author_id", "Unknown")
|
||||
|
||||
is_uploader = comment.get("author_is_uploader", False)
|
||||
|
||||
cleaned_comment = {
|
||||
"comment_id": comment["id"],
|
||||
"comment_text": comment["text"].replace("\xa0", ""),
|
||||
"comment_timestamp": comment["timestamp"],
|
||||
"comment_time_text": time_text,
|
||||
"comment_likecount": comment.get("like_count", None),
|
||||
"comment_is_favorited": comment.get("is_favorited", False),
|
||||
"comment_author": comment["author"],
|
||||
"comment_author_id": comment["author_id"],
|
||||
"comment_author_is_uploader": is_uploader,
|
||||
"comment_author_thumbnail": comment["author_thumbnail"],
|
||||
"comment_author_is_uploader": comment.get(
|
||||
"author_is_uploader", False
|
||||
),
|
||||
"comment_id": comment["id"],
|
||||
"comment_is_favorited": comment.get("is_favorited", False),
|
||||
"comment_likecount": comment.get("like_count", None),
|
||||
"comment_parent": comment["parent"],
|
||||
"comment_text": comment["text"].replace("\xa0", ""),
|
||||
"comment_time_text": time_text,
|
||||
"comment_timestamp": comment["timestamp"],
|
||||
}
|
||||
|
||||
return cleaned_comment
|
||||
|
|
|
|||
Loading…
Reference in New Issue