widelands-dev team mailing list archive

[kaputtnik <somal@xxxxxxxx>]

kaputtnik has proposed merging lp:~widelands-dev/widelands-website/anti_spam_3 into lp:widelands-website.

Requested reviews:
  Widelands Developers (widelands-dev)
Related bugs:
  Bug #1614403 in Widelands Website: "Ideas to prevent spammers, make their work harder"
  https://bugs.launchpad.net/widelands-website/+bug/1614403

For more details, see:
https://code.launchpad.net/~widelands-dev/widelands-website/anti_spam_3/+merge/309167

A bunch of changes:

- Moved the spam check from pybb.views.py to pybb.forms.py
- Prevent e-mails and notifications to users if spam is detected
- Instead inform settings.ADMINS per E-Mail of potential spam. The E-mail contains the topic name and the whole post with a link to the admin page pybb/post
- Importand: Deleting a post over the admin page pybb/post uses now the method of the model instead of Django's default action. This prevents index errors if a topic has no post.
- Added a method to pybb.Post to unhide posts. This is currently callable only over the admin site but having this in pybb.Post makes it available over a button in the posts view (just like 'Stick topic' or similar)
- Changed the views in the admin pages as follows:
   - Added hidden property to the list of pybb/post
   - Added post_count to list in pybb/topic
   - Added the above admin actions to pybb/post
- If an ADMIN decides of no spam to a hidden post and unhide it over the admin action 'Unhide post and inform the users', the usual notices and E-mails are send when the post got unhided.
- Added a regex for International phone numbers and used it to search for in topic_name and post_body. The regex was made by janus and tested against the gathered spam-posts. Thanks janus :-)

-- 
Your team Widelands Developers is requested to review the proposed merge of lp:~widelands-dev/widelands-website/anti_spam_3 into lp:widelands-website.

=== modified file 'mainpage/views.py'
--- mainpage/views.py	2016-08-07 18:35:30 +0000
+++ mainpage/views.py	2016-10-24 20:00:09 +0000
@@ -37,7 +37,6 @@
             recipients = []
             for recipient in INQUIRY_RECIPIENTS:
                 recipients.append(recipient[1])
-                print('recipeients: ', recipients)
 
             send_mail(subject, message, sender,
                       recipients, fail_silently=False)

=== modified file 'pybb/admin.py'
--- pybb/admin.py	2016-10-13 15:50:48 +0000
+++ pybb/admin.py	2016-10-24 20:00:09 +0000
@@ -3,6 +3,23 @@
 from django.contrib import admin
 from pybb.models import Category, Forum, Topic, Post, Read
 
+def delete_selected(modeladmin, request, queryset):
+    """ Overwritten Django's default action to delete a post.
+    
+    This action uses the delete() method of the post model.
+    This ensures also deleting a topic if neccesary, preventing
+    index-errors if a topic has only one post.
+    """
+    for obj in queryset:
+        obj.delete()
+delete_selected.short_description = 'Delete selected posts'
+
+def unhide_post(modeladmin, request, queryset):
+    "Unhide post(s) and inform subscribers"
+    for obj in queryset:
+        obj.unhide_post()
+unhide_post.short_description = 'Unhide post and inform subscribers'
+
 class CategoryAdmin(admin.ModelAdmin):
     list_display = ['name', 'position', 'forum_count']
     list_per_page = 20
@@ -27,7 +44,7 @@
         )
 
 class TopicAdmin(admin.ModelAdmin):
-    list_display = ['name', 'forum', 'created', 'head']
+    list_display = ['name', 'forum', 'created', 'head', 'is_hidden']
     list_per_page = 20
     ordering = ['-created']
     date_hierarchy = 'created'
@@ -45,19 +62,20 @@
         )
 
 class PostAdmin(admin.ModelAdmin):
-    list_display = ['topic', 'user', 'created', 'updated', 'summary']
+    list_display = ['summary', 'topic', 'user', 'created', 'hidden']
     list_per_page = 20
     ordering = ['-created']
     date_hierarchy = 'created'
     search_fields = ['body']
+    actions = [delete_selected, unhide_post]
     fieldsets = (
         (None, {
-                'fields': ('topic', 'user', 'markup')
+                'fields': ('topic', 'user', 'markup', 'hidden')
                 }
          ),
         (_('Additional options'), {
                 'classes': ('collapse',),
-                'fields' : (('created', 'updated'), 'user_ip', 'hidden')
+                'fields' : (('created', 'updated'), 'user_ip' )
                 }
          ),
         (_('Message'), {

=== modified file 'pybb/forms.py'
--- pybb/forms.py	2016-10-03 14:10:48 +0000
+++ pybb/forms.py	2016-10-24 20:00:09 +0000
@@ -9,8 +9,12 @@
 
 from pybb.models import Topic, Post, PrivateMessage, Attachment
 from pybb import settings as pybb_settings
-
+from django.conf import settings
 from notification.models import send
+from django.core.mail import send_mail
+from django.contrib.sites.models import Site
+
+INTERN_PHONE_NR = re.compile('([0]{2}|[\+\@\ ]{1})\d{2}[\ \-\=]{,1}\d{8,11}')
 
 class AddPostForm(forms.ModelForm):
     name = forms.CharField(label=_('Subject'))
@@ -63,20 +67,48 @@
             topic_is_new = False
             topic = self.topic
 
+        # Check for spam
+        # TODO: This is currently a simple keyword search. Maybe add akismet check here
+        hidden = False
+        text = self.cleaned_data['body']
+        if any(x in text.lower() for x in settings.ANTI_SPAM_BODY):
+            hidden = True
+
+        if re.search(INTERN_PHONE_NR, text):
+            hidden = True
+        
+        if topic_is_new:
+            text = self.cleaned_data['name']
+            if any(x in text.lower() for x in settings.ANTI_SPAM_TOPIC):
+                hidden = True
+            if re.search(INTERN_PHONE_NR, text):
+                hidden = True
+
         post = Post(topic=topic, user=self.user, user_ip=self.ip,
                     markup=self.cleaned_data['markup'],
-                    body=self.cleaned_data['body'])
+                    body=self.cleaned_data['body'], hidden=hidden)
         post.save(*args, **kwargs)
 
         if pybb_settings.ATTACHMENT_ENABLE:
             self.save_attachment(post, self.cleaned_data['attachment'])
 
-        if topic_is_new:
-            send(User.objects.all(), "forum_new_topic",
-                {'topic': topic, 'post':post, 'user':topic.user})
+        if not hidden:
+            if topic_is_new:
+                send(User.objects.all(), "forum_new_topic",
+                    {'topic': topic, 'post':post, 'user':topic.user})
+            else:
+                send(self.topic.subscribers.all(), "forum_new_post",
+                    {'post':post, 'topic':topic, 'user':post.user})
         else:
-            send(self.topic.subscribers.all(), "forum_new_post",
-                {'post':post, 'topic':topic, 'user':post.user})
+            # Inform admins of a hidden post
+            recipients = [addr[1] for addr in settings.ADMINS]
+            message = '\n'.join(['Hidden post:',
+                                 'Topic name: ' + topic.name,
+                                 'Post body: ' + post.body,
+                                 'Admin page: http://'+ Site.objects.get_current().domain + '/admin/login/?next=/admin/pybb/post/'+ str(post.id)])
+            send_mail('A post was hidden by spam check', message, 'pybb@xxxxxxxxxxxxx',
+                      recipients, fail_silently=False)
+
         return post
 
 

=== modified file 'pybb/models.py'
--- pybb/models.py	2016-10-08 09:48:25 +0000
+++ pybb/models.py	2016-10-24 20:00:09 +0000
@@ -15,6 +15,8 @@
 from pybb import settings as pybb_settings
 
 from django.conf import settings
+from notification.models import send
+from django.contrib.auth.models import User
 if settings.USE_SPHINX:
     from djangosphinx.models import SphinxSearch
 
@@ -260,6 +262,18 @@
     def get_absolute_url(self):
         return reverse('pybb_post', args=[self.id])
 
+    def unhide_post(self):
+        "Unhide post(s) and inform subscribers"
+        self.hidden = False
+        self.save()
+        if self.topic.post_count == 1:
+            # The topic is new
+            send(User.objects.all(), "forum_new_topic",
+                 {'topic': self.topic, 'post':self, 'user':self.topic.user})
+        else:
+            # Inform topic subscribers
+            send(self.topic.subscribers.all(), "forum_new_post",
+                    {'post':self, 'topic':self.topic, 'user':self.user})
 
     def delete(self, *args, **kwargs):
         self_id = self.id

=== modified file 'pybb/views.py'
--- pybb/views.py	2016-10-12 20:42:21 +0000
+++ pybb/views.py	2016-10-24 20:00:09 +0000
@@ -18,7 +18,7 @@
 from pybb.forms import AddPostForm, EditPostForm, UserSearchForm
 from pybb import settings as pybb_settings
 from pybb.orm import load_related
-from django.conf import settings
+
 from wl_utils import get_real_ip
 
 try:
@@ -161,31 +161,14 @@
                       initial={'markup': "markdown", 'body': quote})
 
     if form.is_valid():
-        # TODO: Add akismet check here
-        spam = False
-
-        # Check in post text.
-        text = form.cleaned_data['body']
-        if any(x in text.lower() for x in settings.ANTI_SPAM_BODY):
-            spam = True
-        
-        # Check in topic subject ('name' is empty if this a post to an existing topic)
-        text = form.cleaned_data['name']
-        if text != '':
-            # This is a new topic
-            if any(x in text.lower() for x in settings.ANTI_SPAM_TOPIC):
-                spam = True
-
         post = form.save()
-        if spam:
-            # Hide the post against normal users
-            post.hidden = True
-            post.save()
+        if not topic:
+            post.topic.subscribers.add(request.user)
+
+        if post.hidden:
             # Redirect to an info page to inform the user
             return HttpResponseRedirect('pybb_moderate_info')
 
-        if not topic:
-            post.topic.subscribers.add(request.user)
         return HttpResponseRedirect(post.get_absolute_url())
 
     if topic: