diff --git a/README.md b/README.md index 6c06ee3..c66b834 100644 --- a/README.md +++ b/README.md @@ -1,420 +1,75 @@ -#!/bin/bash +# M365 Mailbox Backup Tool (self-hosted-m365-backup) -#================================================================================= -# M365 Mailbox Backup Tool - Automatic Installation Script (v2.1 - Interactive IP) -#================================================================================= +This is an open-source, self-hosted web application designed to back up Microsoft 365 mailboxes to a local server in the standard `.eml` format. Built with a security-first approach, it leverages the Microsoft Identity Platform for authentication, ensuring that access is protected by your organization's existing security policies. -# --- Color Codes --- -GREEN='\033[0;32m' -RED='\033[0;31m' -NC='\033[0m' # No Color +## Features -if [ "$EUID" -ne 0 ]; then - echo -e "${RED}Please run this script with 'sudo': sudo ./install.sh${NC}" - exit 1 -fi +- **Web-Based Management Panel:** A modern and convenient web interface for all operations. +- **Secure, Enterprise-Grade Authentication:** Instead of managing local passwords, the application integrates with the Microsoft Identity Platform ("Login with Microsoft"). This ensures that your organization's existing security policies, including **Multi-Factor Authentication (2FA/MFA)**, are automatically enforced, providing a highly secure login experience for administrators. +- **Selective Backup:** Choose one or more users to back up from a dynamically loaded list of all tenant users. +- **Date Filtering:** Back up items from specific time ranges (presets or custom dates). +- **Asynchronous Background Tasks:** Large backup jobs run in the background using Celery and Redis, ensuring the UI is always responsive. +- **Task Status Dashboard:** Monitor the status of all initiated backup tasks (Pending, In Progress, Success, Failure). +- **Interactive Restore Portal:** An Outlook-like three-pane interface to browse backed-up data: + - Navigation through the original mailbox folder structure (Inbox, Sent Items, etc.). + - A list of emails within the selected folder. + - In-browser preview of emails without downloading. +- **Privacy-by-Design Filter:** The email preview pane is blurred by default to respect user privacy, with a hover-to-reveal feature for administrators. +- **Full-Text Search:** Search within the subject and body of all backed-up emails for a specific user. +- **Automated Installation:** A single `install.sh` script to deploy the entire application and its dependencies on a fresh Ubuntu server. -echo "=== M365 Backup Tool Installation Started ===" +## Architecture -# --- NEW: INTERACTIVE IP PROMPT --- -DETECTED_IP=$(hostname -I | awk '{print $1}') -read -p "Enter the server IP address or press Enter to use the detected IP [$DETECTED_IP]: " USER_IP -IP_ADDR=${USER_IP:-$DETECTED_IP} -# ------------------------------------ +- **Backend:** Python, Flask +- **Web Server:** Gunicorn, Nginx (as a Reverse Proxy) +- **Background Tasks:** Celery +- **Queue Manager / Broker:** Redis +- **Frontend:** HTML, Pico.css, HTMX +- **Authentication:** Microsoft Identity Platform (OAuth 2.0) -REDIRECT_URI="https://$IP_ADDR/get_token" -echo "---------------------------------------------------------------------" -echo "IMPORTANT: Please ensure you have set the following Redirect URI" -echo "in your Azure AD App Registration under the 'Web' platform section:" -echo "" -echo "$REDIRECT_URI" -echo "" -echo "---------------------------------------------------------------------" -read -p "Press Enter after you have added the Redirect URI in the Azure Portal..." +## Installation Guide -echo -n "Enter your Azure Tenant ID: " -read TENANT_ID -echo -n "Enter your Application (Client) ID: " -read CLIENT_ID -echo -n "Enter your Client Secret Value (will not be visible): " -read -s CLIENT_SECRET -echo "" -echo -n "Enter comma-separated emails of allowed admins: " -read ALLOWED_ADMINS -echo "" -echo "Credentials received." +The installation process consists of two main parts: preparing the prerequisites in the Azure Portal and then running the installation script on your server. -# --- System Dependency Installation --- -echo "--> Updating system and installing required packages..." -apt-get update > /dev/null 2>&1 -apt-get install -y nginx redis-server python3-pip python3-venv > /dev/null 2>&1 +### I. Azure Portal Configuration (Prerequisites) -# --- Project Directory and Virtual Environment --- -PROJECT_DIR="/opt/m365-backup-tool" -USERNAME=${SUDO_USER:-$(whoami)} +Before running the installation script, you must complete the following steps in the Azure Portal. -echo "--> Creating project directory: $PROJECT_DIR" -mkdir -p "$PROJECT_DIR" -cd "$PROJECT_DIR" +**1. App Registration:** + - Log in to `portal.azure.com`. + - Navigate to **Azure Active Directory** > **App registrations** > **+ New registration**. + - Give your application a name (e.g., `M365-Backup-Tool`) and click **Register**. -echo "--> Creating Python virtual environment..." -python3 -m venv venv +**2. API Permissions:** + - In your newly created app, navigate to **API permissions** from the left menu. + - Click **+ Add a permission** > **Microsoft Graph**. + - You will need to add two types of permissions: + - **Application permissions:** + - `User.Read.All` (To get the list of all users for backup selection) + - `Mail.Read.All` (To read all mailboxes for the backup process) + - **Delegated permissions:** + - `User.Read` (To allow users to log in to the application with their Microsoft account) + - After adding these permissions, you **must** click the **"Grant admin consent for [Your Organization's Name]"** button to approve them. The status for all permissions should show a green checkmark and "Granted". -echo "--> Installing Python libraries..." -source venv/bin/activate -pip install Flask gunicorn "celery[redis]" msal requests python-decouple > /dev/null 2>&1 -deactivate +**3. Add a Redirect URI:** + - From the left menu, select **Authentication**. + - Click **+ Add a platform** > **Web**. + - In the **Redirect URIs** field, you will enter a URI in the format `https:///get_token`. The `install.sh` script will show you the exact URI to use when you run it. + - Click **Save**. -# --- Application File Creation --- -echo "--> Creating .env file for credentials..." -cat < .env -TENANT_ID=${TENANT_ID} -CLIENT_ID=${CLIENT_ID} -CLIENT_SECRET=${CLIENT_SECRET} -FLASK_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_hex(24))') -ALLOWED_ADMINS=${ALLOWED_ADMINS} -REDIRECT_URI=${REDIRECT_URI} -EOF +**4. Create a Client Secret:** + - From the left menu, select **Certificates & secrets** > **+ New client secret**. + - Add a description and choose an expiration period (e.g., 24 months). + - Click **Add**. + - **VERY IMPORTANT:** Immediately copy the **"Value"** of the newly created secret and save it somewhere safe. You will not be able to see this value again after you leave the page. -echo "--> Creating application files and templates..." -# app.py file -cat <<'EOF' > app.py -import os, re, msal, requests, json, secrets -from flask import Flask, request, redirect, url_for, render_template, send_from_directory, session -from celery import Celery -from email import policy -from email.parser import BytesParser -from datetime import datetime, timedelta -from decouple import config -from functools import wraps +**Summary:** At the end of this stage, you must have these 3 pieces of information: **Tenant ID**, **Client ID**, and **Client Secret Value**. -TENANT_ID = config('TENANT_ID', default='') -CLIENT_ID = config('CLIENT_ID', default='') -CLIENT_SECRET = config('CLIENT_SECRET', default='') -FLASK_SECRET_KEY = config('FLASK_SECRET_KEY') -ALLOWED_ADMINS = [admin.strip().lower() for admin in config('ALLOWED_ADMINS', default='').split(',')] -REDIRECT_URI = config('REDIRECT_URI') -AUTHORITY = f"https://login.microsoftonline.com/{TENANT_ID}" -SCOPE = ["User.Read"] -BACKUP_DIR = os.path.join(os.getcwd(), "backup") +### II. Server Installation -app = Flask(__name__) -app.secret_key = FLASK_SECRET_KEY -app.config.update(broker_url='redis://localhost:6379/0', result_backend='redis://localhost:6379/0') -celery = Celery(app.name, broker=app.config['broker_url']) -celery.conf.update(app.config) +Log into your fresh **Ubuntu 22.04 LTS** server. The following commands will download the installation script, make it executable, and then run it with the necessary privileges. -msal_app = msal.ConfidentialClientApplication(client_id=CLIENT_ID, authority=AUTHORITY, client_credential=CLIENT_SECRET) - -def login_required(f): - @wraps(f) - def decorated_function(*args, **kwargs): - if 'user' not in session: return redirect(url_for('show_login_page')) - if session['user'].get('email', '').lower() not in ALLOWED_ADMINS: - session.clear(); return "Access Denied. You are not an authorized administrator.", 403 - return f(*args, **kwargs) - return decorated_function - -@app.route('/login_page') -def show_login_page(): return render_template('login.html') - -@app.route('/login') -def login(): - session["flow"] = msal_app.initiate_auth_code_flow(SCOPE, redirect_uri=REDIRECT_URI) - return redirect(session["flow"]["auth_uri"]) - -@app.route('/get_token') -def get_token(): - try: - result = msal_app.acquire_token_by_auth_code_flow(session.get("flow", {}), request.args) - if "error" in result: return f"Login Error: {result.get('error_description')}", 400 - claims = result.get("id_token_claims", {}) - user_email = claims.get("preferred_username") - if not user_email or user_email.lower() not in ALLOWED_ADMINS: - session.clear(); return "Access Denied. Your account is not in the list of authorized administrators.", 403 - session["user"] = {"name": claims.get("name"), "email": user_email} - except ValueError as e: return f"Error processing token: {str(e)}", 400 - return redirect(url_for('index')) - -@app.route('/logout') -def logout(): - session.clear() - return redirect(f"https://login.microsoftonline.com/common/oauth2/v2.0/logout?post_logout_redirect_uri={url_for('show_login_page', _external=True, _scheme='https')}") - -def sanitize_filename(filename): return re.sub(r'[\\/*?:"<>|]', "", filename) - -@celery.task -def trigger_backup(user_email, date_option='all_time', start_date_str=None, end_date_str=None): - app_for_backup = msal.ConfidentialClientApplication(client_id=CLIENT_ID, authority=AUTHORITY, client_credential=CLIENT_SECRET) - result = app_for_backup.acquire_token_for_client(scopes=["https://graph.microsoft.com/.default"]) - if "access_token" not in result: return f"Auth Error for background task: {result.get('error_description')}" - access_token = result['access_token']; headers = {'Authorization': 'Bearer ' + access_token} - user_backup_path = os.path.join(BACKUP_DIR, user_email) - if not os.path.exists(user_backup_path): os.makedirs(user_backup_path) - all_folders, folders_url = [], f"https://graph.microsoft.com/v1.0/users/{user_email}/mailFolders?$top=100" - while folders_url: - response = requests.get(folders_url, headers=headers) - if response.status_code != 200: return f"Error fetching mail folders: {response.json()}" - data = response.json(); all_folders.extend(data.get('value', [])); folders_url = data.get('@odata.nextLink') - total_message_count = 0 - for folder in all_folders: - folder_name, folder_id = folder.get('displayName'), folder.get('id') - if not folder_name or not folder_id: continue - local_folder_path = os.path.join(user_backup_path, sanitize_filename(folder_name)) - if not os.path.exists(local_folder_path): os.makedirs(local_folder_path) - filter_query = "" - now = datetime.utcnow() - if date_option == 'last_3_months': filter_query = f"$filter=receivedDateTime ge {(now - timedelta(days=90)).isoformat()}Z" - elif date_option == 'last_6_months': filter_query = f"$filter=receivedDateTime ge {(now - timedelta(days=180)).isoformat()}Z" - elif date_option == 'last_1_year': filter_query = f"$filter=receivedDateTime ge {(now - timedelta(days=365)).isoformat()}Z" - elif date_option == 'last_2_years': filter_query = f"$filter=receivedDateTime ge {(now - timedelta(days=730)).isoformat()}Z" - elif date_option == 'custom_range' and start_date_str and end_date_str: - start_iso = datetime.fromisoformat(start_date_str).isoformat(); end_iso = (datetime.fromisoformat(end_date_str) + timedelta(days=1)).isoformat() - filter_query = f"$filter=receivedDateTime ge {start_iso}Z and receivedDateTime lt {end_iso}Z" - base_url = f"https://graph.microsoft.com/v1.0/users/{user_email}/mailFolders/{folder_id}/messages" - messages_url = f"{base_url}?$top=1000" - if filter_query: messages_url = f"{base_url}?{filter_query}&$top=1000" - message_count_in_folder = 0 - while messages_url: - msg_response = requests.get(messages_url, headers=headers) - if msg_response.status_code != 200: break - messages_data = msg_response.json(); messages = messages_data.get('value', []) - for message in messages: - message_id = message.get('id') - if not message_id: continue - mime_url = f"https://graph.microsoft.com/v1.0/users/{user_email}/messages/{message_id}/$value" - mime_response = requests.get(mime_url, headers=headers) - if mime_response.status_code == 200: - with open(os.path.join(local_folder_path, f"{sanitize_filename(message_id)}.eml"), 'wb') as f: f.write(mime_response.content) - message_count_in_folder += 1 - messages_url = messages_data.get('@odata.nextLink') - total_message_count += message_count_in_folder - return f"Total {total_message_count} emails downloaded." -@app.route('/') -@login_required -def index(): return render_template('index.html') -@app.route('/get_users') -@login_required -def get_users(): - app_for_users = msal.ConfidentialClientApplication(client_id=CLIENT_ID, authority=AUTHORITY, client_credential=CLIENT_SECRET) - result = app_for_users.acquire_token_for_client(scopes=["https://graph.microsoft.com/.default"]) - if "access_token" not in result: return render_template('partials/user_list_error.html', error_message="App Authentication Error.") - headers = {'Authorization': 'Bearer ' + result['access_token']} - users, users_url = [], "https://graph.microsoft.com/v1.0/users?$select=displayName,userPrincipalName" - while users_url: - response = requests.get(users_url, headers=headers) - if response.status_code != 200: return render_template('partials/user_list_error.html', error_message="Error fetching users.") - data = response.json(); users.extend(data.get('value', [])); users_url = data.get('@odata.nextLink') - return render_template('partials/user_list.html', users=sorted(users, key=lambda u: u.get('displayName') or '')) -@app.route('/backup', methods=['POST']) -@login_required -def backup(): - emails = request.form.getlist('emails'); date_option = request.form.get('date_option'); start_date = request.form.get('start_date'); end_date = request.form.get('end_date') - if 'tasks' not in session: session['tasks'] = [] - for email in emails: - task = trigger_backup.delay(email, date_option, start_date, end_date) - session['tasks'].append({'id': task.id, 'email': email}) - session.modified = True - return redirect(url_for('list_tasks')) -@app.route('/tasks') -@login_required -def list_tasks(): - task_info = [] - for task_meta in session.get('tasks', []): - task = trigger_backup.AsyncResult(task_meta.get('id')) - task_info.append({'id': task_meta.get('id'), 'email': task_meta.get('email'), 'status': task.state, 'result': task.result if task.state in ['SUCCESS', 'FAILURE'] else str(task.info)}) - return render_template('tasks.html', tasks=reversed(task_info)) -@app.route('/backups') -@login_required -def list_backups(): - if not os.path.exists(BACKUP_DIR): os.makedirs(BACKUP_DIR) - return render_template('backups.html', users=os.listdir(BACKUP_DIR)) -@app.route('/backups/') -@login_required -def restore_portal(user_email): - user_dir = os.path.join(BACKUP_DIR, user_email) - if not os.path.exists(user_dir): return "No backup found for this user.", 404 - all_folders = [d for d in os.listdir(user_dir) if os.path.isdir(os.path.join(user_dir, d))] - active_folder_name = request.args.get('folder') - if not active_folder_name: active_folder_name = next((f for f in ['Inbox'] if f in all_folders), all_folders[0] if all_folders else None) - search_query = request.args.get('q', '').lower() - emails = [] - if active_folder_name: - selected_folder_path = os.path.join(user_dir, active_folder_name) - if os.path.exists(selected_folder_path): - parser = BytesParser(policy=policy.default) - for filename in os.listdir(selected_folder_path): - if filename.endswith(".eml"): - with open(os.path.join(selected_folder_path, filename), 'rb') as f: msg = parser.parse(f) - if search_query: - content_to_search = str(msg.get_payload(decode=True)) - if msg['subject']: content_to_search += msg['subject'] - if search_query not in content_to_search.lower(): continue - emails.append({'filename': filename, 'subject': msg['subject'], 'from': msg['from'], 'date': msg['date']}) - return render_template('portal.html', user_email=user_email, folders=sorted(all_folders), emails=emails, active_folder=active_folder_name, search_query=search_query) -@app.route('/download///') -@login_required -def download_file(user_email, folder_name, filename): return send_from_directory(directory=os.path.join(BACKUP_DIR, user_email, folder_name), path=filename, as_attachment=True) -@app.route('/preview///') -@login_required -def preview_file(user_email, folder_name, filename): - file_path = os.path.join(BACKUP_DIR, user_email, folder_name, filename) - if not os.path.exists(file_path): return {"error": "File not found"}, 404 - with open(file_path, 'rb') as f: msg = BytesParser(policy=policy.default).parse(f) - html_body, plain_body = None, None - if msg.is_multipart(): - for part in msg.walk(): - ct = part.get_content_type() - if ct == 'text/html' and not html_body: html_body = part.get_payload(decode=True).decode(part.get_content_charset() or 'utf-8', errors='ignore') - elif ct == 'text/plain' and not plain_body: plain_body = part.get_payload(decode=True).decode(part.get_content_charset() or 'utf-8', errors='ignore') - else: plain_body = msg.get_payload(decode=True).decode(msg.get_content_charset() or 'utf-8', errors='ignore') - if html_body: return {"body": html_body} - elif plain_body: return {"body": f"
{plain_body}
"} - else: return {"body": "Could not display content."} -EOF - -mkdir -p templates/partials -cat <<'EOF' > templates/login.html -Login - M365 Backup Tool
-EOF -cat <<'EOF' > templates/layout.html -{% block title %}{% endblock %} - M365 Backup Tool
{% block content %}{% endblock %}
-EOF -cat <<'EOF' > templates/index.html -{% extends 'layout.html' %}{% block title %}New Backup{% endblock %}{% block content %}

Start New Backup

Select users to back up:


Loading user list from Azure...

{% endblock %} -EOF -cat <<'EOF' > templates/partials/user_list.html -
-{% for user in users %} -{% else %}{% endfor %} -
SelectDisplay NameEmail
{{ user.displayName }}{{ user.userPrincipalName }}
No users found or error loading users.
-EOF -cat <<'EOF' > templates/partials/user_list_error.html -
Error Loading Users

Could not load the user list.

Details: {{ error_message }}

-EOF -cat <<'EOF' > templates/backups.html -{% extends 'layout.html' %}{% block title %}Existing Backups{% endblock %}{% block content %}

Backed Up Users

    {% for user in users %}
  • {{ user }}
    • {% else %}
  • No backups found yet.
    • {% endfor %}
{% endblock %} -EOF -cat <<'EOF' > templates/portal.html -{% extends 'layout.html' %}{% block title %}{{ user_email }} - Restore Portal{% endblock %}{% block content %} - -

Restore Portal for: {{ user_email }}

Select an email to preview...

-{% endblock %} -EOF -cat <<'EOF' > templates/tasks.html -{% extends 'layout.html' %}{% block title %}Task Status{% endblock %}{% block content %}

Task Status

-{% for task in tasks %} -{% else %}{% endfor %} -
Task IDUserStatusResult
{{ task.id }}{{ task.email }}{{ task.status }}{{ task.result }}
No tasks have been started yet.
Refresh Page{% endblock %} -EOF - -echo "--> Creating self-signed SSL certificate..." -mkdir -p /etc/nginx/ssl -openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ - -keyout /etc/nginx/ssl/m365-backup.key \ - -out /etc/nginx/ssl/m365-backup.crt \ - -subj "/C=XX/ST=Global/L=Global/O=Global/OU=IT/CN=$IP_ADDR" - -echo "--> Creating systemd service files..." -cat < /etc/systemd/system/m365-web.service -[Unit] -Description=Gunicorn for M365 Backup App -After=network.target -[Service] -User=$USERNAME -Group=www-data -WorkingDirectory=$PROJECT_DIR -ExecStart=$PROJECT_DIR/venv/bin/gunicorn --workers 3 --bind unix:$PROJECT_DIR/m365-backup.sock -m 007 app:app -Restart=always -[Install] -WantedBy=multi-user.target -EOF - -cat < /etc/systemd/system/m365-worker.service -[Unit] -Description=Celery Worker for M365 Backup App -After=network.target -[Service] -User=$USERNAME -Group=www-data -WorkingDirectory=$PROJECT_DIR -ExecStart=$PROJECT_DIR/venv/bin/celery -A app.celery worker --loglevel=info -Restart=always -[Install] -WantedBy=multi-user.target -EOF - -echo "--> Creating Nginx proxy configuration..." -cat < /etc/nginx/sites-available/m365-backup -server { - listen 80; - server_name $IP_ADDR; - return 301 https://\$server_name\$request_uri; -} -server { - listen 443 ssl; - server_name $IP_ADDR; - ssl_certificate /etc/nginx/ssl/m365-backup.crt; - ssl_certificate_key /etc/nginx/ssl/m365-backup.key; - ssl_protocols TLSv1.2 TLSv1.3; - location / { - include proxy_params; - proxy_pass http://unix:$PROJECT_DIR/m365-backup.sock; - } -} -EOF -rm -f /etc/nginx/sites-enabled/default -ln -sf /etc/nginx/sites-available/m365-backup /etc/nginx/sites-enabled/ - -chown -R $USERNAME:www-data "$PROJECT_DIR" -chmod -R 775 "$PROJECT_DIR" - -echo "--> Starting and enabling all services..." -systemctl daemon-reload -systemctl restart redis-server nginx -systemctl start m365-web m365-worker -systemctl enable redis-server nginx m365-web m365-worker - -echo "--> Verifying service status..." -sleep 5 -FINAL_STATUS=0 -if systemctl is-active --quiet m365-web.service; then - echo -e "${GREEN}✅ Web Server (Gunicorn) is running.${NC}" -else - echo -e "${RED}❌ Web Server (Gunicorn) failed to start!${NC}"; FINAL_STATUS=1; fi -if systemctl is-active --quiet m365-worker.service; then - echo -e "${GREEN}✅ Background Worker (Celery) is running.${NC}" -else - echo -e "${RED}❌ Background Worker (Celery) failed to start!${NC}"; FINAL_STATUS=1; fi - -echo "" -echo "============================================================" -if [ $FINAL_STATUS -eq 0 ]; then - echo -e "${GREEN}✅ INSTALLATION COMPLETED SUCCESSFULLY!${NC}" - echo "" - echo "You can access the application at: https://$IP_ADDR" -else - echo -e "${RED}❌ INSTALLATION FAILED!${NC}" - echo -e "${RED}Please check the error messages and service logs.${NC}" -fi -echo "============================================================" +```bash +wget [https://raw.githubusercontent.com/DualStackAdmin/self-hosted-m365-backup/main/install.sh](https://raw.githubusercontent.com/DualStackAdmin/self-hosted-m365-backup/main/install.sh) +chmod +x install.sh +sudo ./install.sh