Hoàn thiện phần mềm crawl data từ website howkteam.com

<Window x:Class="CrawlDataFromHowKteam.MainWindow"
        xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
        xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
        xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
        xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
        xmlns:local="clr-namespace:CrawlDataFromHowKteam"
        mc:Ignorable="d"
        Title="MainWindow" 
        WindowState="Maximized"
        >
    <Grid>
        <Grid.RowDefinitions>
            <RowDefinition Height="auto"></RowDefinition>
            <RowDefinition></RowDefinition>
        </Grid.RowDefinitions>
        <Grid.ColumnDefinitions>
            <ColumnDefinition Width="250" MinWidth="250"></ColumnDefinition>
            <ColumnDefinition Width="5"></ColumnDefinition>
            <ColumnDefinition MinWidth="200"></ColumnDefinition>
        </Grid.ColumnDefinitions>
        <Grid Grid.Column="0" Grid.Row="0">
            <Button Click="Button_Click_1">Load data</Button>
        </Grid>       

        <GridSplitter HorizontalAlignment="Stretch" Grid.Column="1" Grid.RowSpan="2"></GridSplitter>
        <TreeView x:Name="treeMain" Grid.Column="0" Grid.Row="1">
            <TreeView.ItemTemplate>
                <HierarchicalDataTemplate ItemsSource="{Binding Items}">

                    <TextBlock Margin="5 5 5 5" PreviewMouseDown="TextBlock_PreviewMouseDown" Foreground="Blue" Text="{Binding Name}" Tag="{Binding URL}"></TextBlock>

                </HierarchicalDataTemplate>
            </TreeView.ItemTemplate>
        </TreeView>
        <WebBrowser Grid.RowSpan="2" Navigated="wbMain_Navigated" x:Name="wbMain" Grid.Column="2" Grid.Row="0"></WebBrowser>
    </Grid>
</Window>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Diagnostics;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Data;
using System.Windows.Documents;
using System.Windows.Input;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Windows.Navigation;
using System.Windows.Shapes;

namespace CrawlDataFromHowKteam
{
    /// <summary>
    /// Interaction logic for MainWindow.xaml
    /// </summary>
    public partial class MainWindow : Window
    {
        #region properties
        ObservableCollection<MenuTreeItem> TreeItems;
        string HomePage = "https://www.howkteam.com/";

        HttpClient httpClient;
        HttpClientHandler handler;
        CookieContainer cookie = new CookieContainer();


        #endregion
        public MainWindow()
        {
            InitializeComponent();

            IniHttpClient();            

            TreeItems = new ObservableCollection<MenuTreeItem>();            
            treeMain.ItemsSource = TreeItems;
        }

        #region methods
        void IniHttpClient()
        {
            handler = new HttpClientHandler
            {
                CookieContainer = cookie,
                ClientCertificateOptions = ClientCertificateOption.Automatic,
                AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
                AllowAutoRedirect = true,
                UseDefaultCredentials = false
            };

            httpClient = new HttpClient(handler);

            //httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) coc_coc_browser/63.4.154 Chrome/57.4.2987.154 Safari/537.36");
            /*
             * Header:
             * - Origin
             * - Host
             * - Referer
             * - :scheme
             * - accept
             * - Accept-Encoding
             * - Accept-Language
             * - User-Argent
             */


            httpClient.BaseAddress = new Uri(HomePage);
        }
        void AddItemIntoTreeViewItem(ObservableCollection<MenuTreeItem> root, MenuTreeItem node)
        {
            treeMain.Dispatcher.Invoke(new Action(()=> {
                root.Add(node);
            }));
        }

        string CrawlDataFromURL(string url)
        {
            string html = "";
            
            html = WebUtility.HtmlDecode(httpClient.GetStringAsync(url).Result);

            //html = httpClient.PostAsync(url,new StringContent("")).Result.Content.ReadAsStringAsync().Result;

            return html;
        }

        void Crawl(string url)
        {
            string htmlLearn = CrawlDataFromURL(url);
            var CourseList = Regex.Matches(htmlLearn, @"<div class=""info-course(.*?)</div>", RegexOptions.Singleline);
            foreach (var course in CourseList)
            {
                string courseName = Regex.Match(course.ToString(), @"(?=<h5>).*?(?=</h5>)").Value.Replace("<h5>","");
                string linkCourse = Regex.Match(course.ToString(), @"'(.*?)'", RegexOptions.Singleline).Value.Replace("'","");

                MenuTreeItem item = new MenuTreeItem();
                item.Name = courseName;
                item.URL = linkCourse;

                AddItemIntoTreeViewItem(TreeItems, item);

                string htmlCourse = CrawlDataFromURL(linkCourse);
                string sideBar = Regex.Match(htmlCourse, @"<div class=""sidebardetail"">(.*?)</ul>", RegexOptions.Singleline).Value;
                var listLecture = Regex.Matches(sideBar, @"<li(.*?)</li>", RegexOptions.Singleline);
                foreach (var lecture in listLecture)
                {
                    string lectureName = Regex.Match(lecture.ToString(), @"<li title=""(.*?)"">", RegexOptions.Singleline).Value.Replace("\">", "").Replace("<li title=\"", "");
                    string linkLecture = Regex.Match(lecture.ToString(), @"<a href=""(.*?)"">", RegexOptions.Singleline).Value.Replace("<a href=\"", "").Replace("\">","");

                    MenuTreeItem Subitem = new MenuTreeItem();
                    Subitem.Name = lectureName;
                    Subitem.URL = linkLecture;
                    AddItemIntoTreeViewItem(item.Items, Subitem);
                }
            }
        }
        public static void SetSilent(WebBrowser browser, bool silent)
        {
            if (browser == null)
                throw new ArgumentNullException("browser");

            // get an IWebBrowser2 from the document
            IOleServiceProvider sp = browser.Document as IOleServiceProvider;
            if (sp != null)
            {
                Guid IID_IWebBrowserApp = new Guid("0002DF05-0000-0000-C000-000000000046");
                Guid IID_IWebBrowser2 = new Guid("D30C1661-CDAF-11d0-8A3E-00C04FC9E26E");

                object webBrowser;
                sp.QueryService(ref IID_IWebBrowserApp, ref IID_IWebBrowser2, out webBrowser);
                if (webBrowser != null)
                {
                    webBrowser.GetType().InvokeMember("Silent", BindingFlags.Instance | BindingFlags.Public | BindingFlags.PutDispProperty, null, webBrowser, new object[] { silent });
                }
            }
        }
        
        [ComImport, Guid("6D5140C1-7436-11CE-8034-00AA006009FA"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
        private interface IOleServiceProvider
        {
            [PreserveSig]
            int QueryService([In] ref Guid guidService, [In] ref Guid riid, [MarshalAs(UnmanagedType.IDispatch)] out object ppvObject);
        }
        #endregion
        private void Button_Click_1(object sender, RoutedEventArgs e)
        {
            //treeMain.Dispatcher.Invoke(new Action(()=> {
            //    Crawl("Learn");
            //}));
            Task t = new Task(()=> { Crawl("Learn"); });
            t.Start();
        }        

        private void wbMain_Navigated(object sender, NavigationEventArgs e)
        {
            SetSilent(wbMain, true);
        }

        private void TextBlock_PreviewMouseDown(object sender, MouseButtonEventArgs e)
        {
            string url = HomePage + (sender as TextBlock).Tag.ToString();
            wbMain.Navigate(url);
            //Process.Start(url);
        }
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178

using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace CrawlDataFromHowKteam
{
    public class MenuTreeItem
    {
        public string Name { get; set; }
        public string URL { get; set; }
        public ObservableCollection<MenuTreeItem> Items { get; set; }
        public MenuTreeItem()
        {
            this.Items = new ObservableCollection<MenuTreeItem>();
        }
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21