Jump to content

MTASA wiki - offline copies / online mirrors


Recommended Posts

  • 1 month later...
  • 5 weeks later...
  • 1 month later...
  • 2 weeks later...

Since CiBeR's link is for some reason down. I took the time and wrote 2 Scripts in C# to export the MTA wiki.

This will download all page names to a file called pages.txt

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Net;
namespace Rextester
{
    public class Program
    {
        public static string DownloadPage(int iteration, string url)
        {
            using(WebClient wc = new WebClient())
            {
                Console.WriteLine("Requesting page: " + System.Web.HttpUtility.HtmlDecode(url));
                
                Uri uri = new Uri("https://wiki.multitheftauto.com" + System.Web.HttpUtility.HtmlDecode(url));
                string page = wc.DownloadString(uri);
                
                //System.IO.File.WriteAllText(iteration + ".txt", page);
                
                try {
                    Regex regexObj = new Regex("<li><a href=\".*?\" title=\".*?\">(.*?)</a></li>");
                    Match matchResults = regexObj.Match(page);
                    while (matchResults.Success) {
                        System.IO.File.AppendAllText("pages.txt", matchResults.Groups[1].Value+Environment.NewLine);
                        // matched text: matchResults.Value
                        // match start: matchResults.Index
                        // match length: matchResults.Length
                        matchResults = matchResults.NextMatch();
                    } 
                } catch (ArgumentException ex) {
                    // Syntax error in the regular expression
                }
                
                Match match = Regex.Match(page, "<a href=\".*?\" title=\"Special:AllPages\">Previous page .*?</a> \\| <a href=\"(.*?)\" title=\"Special:AllPages\">Next page .*?</a>");
                if(match.Success)
                {
                    Console.WriteLine("Got next page: " + match.Groups[1].Captures[0].Value);
                    return match.Groups[1].Captures[0].Value;
                }else{
                    Console.WriteLine("Failed to download or last page (" + url + ")!");
                }
                
                return "";
            }
        }
        
        public static void Main(string[] args)
        {
            using(WebClient wc = new WebClient())
            {
                string indexPage = wc.DownloadString("https://wiki.multitheftauto.com/index.php?title=Special:AllPages");
                Match match = Regex.Match(indexPage, "<a href=\"(.*?)\" title=\"Special:AllPages\">Next page .*?</a>");
                if(match.Success)
                {
                    string nextPage = match.Groups[1].Captures[0].Value;
                    int iteration = 0;
                    while(nextPage != "")
                    {
                        string prev = nextPage;
                        nextPage = DownloadPage(iteration, nextPage);
                        Console.WriteLine("Got: " + nextPage + " | PrevPage: " + prev);
                        iteration++;
                    }
                }else{
                    Console.WriteLine("Failed to download index page!");
                    Console.WriteLine(indexPage);
                }
            }
        }
    }
}

 

This will parse the pages.txt and download a file called export.xml which can be re-imported into a mediawiki:

using System;
using System.Text;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading;
using System.Threading.Tasks;

namespace Rextester
{

	public class MyFormUrlEncodedContent : ByteArrayContent
	{
		public MyFormUrlEncodedContent(IEnumerable<KeyValuePair<string, string>> nameValueCollection)
			: base(MyFormUrlEncodedContent.GetContentByteArray(nameValueCollection))
		{
			base.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded");
		}
		private static byte[] GetContentByteArray(IEnumerable<KeyValuePair<string, string>> nameValueCollection)
		{
			if (nameValueCollection == null)
			{
				throw new ArgumentNullException("nameValueCollection");
			}
			StringBuilder stringBuilder = new StringBuilder();
			foreach (KeyValuePair<string, string> current in nameValueCollection)
			{
				if (stringBuilder.Length > 0)
				{
					stringBuilder.Append('&');
				}

				stringBuilder.Append(MyFormUrlEncodedContent.Encode(current.Key));
				stringBuilder.Append('=');
				stringBuilder.Append(MyFormUrlEncodedContent.Encode(current.Value));
			}
			return Encoding.Default.GetBytes(stringBuilder.ToString());
		}
		private static string Encode(string data)
		{
			if (string.IsNullOrEmpty(data))
			{
				return string.Empty;
			}
			return System.Net.WebUtility.UrlEncode(data).Replace("%20", "+");
		}
	}
	
    public class Program
    {        
        public static void Main(string[] args)
        {
			Task.Run(async () =>
			{
				using (var client = new HttpClient())
				{
					string text = System.IO.File.ReadAllText("pages.txt");
					
					var values = new Dictionary<string, string>
					{
						{"catname", ""},
						{"pages", text},
						{"curonly", "1"},
						{"templates", "1"},
						{"wpDownload", "1"},
						{"wpEditToken", "%2B%5C"},
						{"title", "Special%3AExport"}
					};
                  
					var content = new MyFormUrlEncodedContent(values);
				
					var response = await client.PostAsync("https://wiki.multitheftauto.com/wiki/Special:Export", content);
				
					var responseString = await response.Content.ReadAsStringAsync();
					System.IO.File.WriteAllText("exported.xml", responseString);
				}
			}).GetAwaiter().GetResult();
        }
    }
}

To convert the xml to sql you can use mwdumper like this:

java -jar mwdumper.jar --format=sql:1.5 exported.xml > sql.sql

 

Edited by KennyKiller
Link to comment

Sorry for that, i'm currently moving to a new hosting.. Some down time expected.

 

EDIT: Site is back online, though it's having some page load timeouts, i'm sorting this out with support team.

EDIT2: Damn microtime, freezes the server. Removed it and now all is working.

Edited by .:CiBeR:.
  • Thanks 1
Link to comment
  • 1 month later...
  • 4 weeks later...
  • 1 year later...

Well, it's been a while since the site went down, and i haven't had the time to get it up and running again, but now i made some time. Regretfully, old backups are lost, if anyone has a backup of the old arkives i'd be happy to re upload them.

The new site is running here: https://nas.globalgaming.xyz/mta/ and will be updated once a month via a cron job.

Hope you'll enjoy.

  • Like 2
  • Thanks 1
Link to comment
  • 1 month later...
On 26/07/2018 at 15:42, .:CiBeR:. said:

Well, it's been a while since the site went down, and i haven't had the time to get it up and running again, but now i made some time. Regretfully, old backups are lost, if anyone has a backup of the old arkives i'd be happy to re upload them.

The new site is running here: https://nas.globalgaming.xyz/mta/ and will be updated once a month via a cron job.

Hope you'll enjoy.

It doesnt have colors or images how to fix it ?

Link to comment
  • 3 weeks later...
  • 4 months later...
  • 1 month later...
  • 8 months later...
  • 3 weeks later...

Create an account or sign in to comment

You need to be a member in order to leave a comment

Create an account

Sign up for a new account in our community. It's easy!

Register a new account

Sign in

Already have an account? Sign in here.

Sign In Now
  • Recently Browsing   0 members

    • No registered users viewing this page.
×
×
  • Create New...