---
tags: 虛擬容錯
---
# openstack nova command新增流程
## 新增HMP -> QMP -> QAPI command
* 要改的file:
* 
* 要**啟動qmp**則需要多加qemu的執行參數：

# qemu-system-TARGET [...] \
    -chardev socket,id=qmp,port=4444,host=localhost,server \
    -mon chardev=qmp,mode=control,pretty=on

* 而**啟動hmp**則需要多加qemu的執行參數：
### 新增qmp指令
* 在qapi/misc.json新增 （註解一定要打）:

##
# @hello-world:
#
# Print a client provided string to the standard output stream.
#
# Since: 2.9
##
{ 'command': 'hello-world', 'data': { '*message': 'str', '*message1': 'str', '*message2': 'str'} }

* 在monitor/qmp-cmds.c新增:
qmp指令的開頭為qmp_;qmp函數的參數都會有對應一個boolean參數開頭為has_

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#define PAGE_SIZE   4096
#define PAGE_SHIFT  12
.....
    
void qmp_hello_world(bool has_message, const char *message, bool has_message1, const char *message1, bool has_message2, const char *message2, Error **errp)
{
    if (has_message && has_message1 && has_message2) {
        printf("%s\n%s\n%s\n", message, message1, message2);
    } 
    else {
        printf("%s\n%s\n%s\n", message, message1, message2);
        printf("Hello, world\n");
    }
}

* 不用去添加函數的原型，因為QAPI是系統自動產生的
* qmp測試：
1. 另外開一個terminal輸入

    $ telnet localhost 4444
      Trying 127.0.0.1...
      Connected to localhost.
      Escape character is '^]'.
      {
          "QMP": {
              "version": {
                  "qemu": {
                      "micro": 50, 
                      "minor": 15, 
                      "major": 0
                  }, 
                  "package": ""
              }, 
              "capabilities": [
              ]
          }
      }

2. 執行{ "execute": "qmp_capabilities" }
3. 執行{ "execute": "hello-world", "arguments": { "message": "We" , "message1": "love" ,"message2": "qemu"} }，terminal會回傳We\nlove\nqemu\n
### 新增hmp指令(human monitor protocal)
* hmp 函數裡頭無法透過system("qmp指令")去完成，因為qmp指令需要透過telnet localhost 4444才能輸入，所以要參考其他 hmp如何去call該qmp的函數，來實做在hmp函數中
* 以create vm template為例
* 在monitor/hmp-cmds.c新增：(hmp-cmds.c不能用global變數所以用函數的static變數實現)

......
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <unistd.h>
#include <sys/types.h>
......
    
//讀取ori_disk_path，這裡用global變數會不管用
char* ori_disk(char *str){
    char *ret;
    ret = strstr(str, "only return");
    static char ori_disk_path_tmp[200]; //只會初始化一次
    if(!ret){
        
        strcpy(ori_disk_path_tmp, str);
    }
    return ori_disk_path_tmp;
}
......
//templating 55 1
//input：vmid, template_num
void hmp_templating(Monitor *mon, const QDict *qdict)
{
    // Input
    const char *vmid = qdict_get_try_str(qdict, "vmid");
    const char *template_num = qdict_get_try_str(qdict, "template_num");
    int template_int = atoi(template_num);
    int template_lastnum = template_int-1;
    char *template_last[100];
    sprintf(template_last, "%d", template_lastnum); //整數轉字串

    // File path
    char state_dir[100] = "/mnt/state";
    char mem_dir[100]= "/mnt/memory";
    char disk_dir[100]= "/mnt/nfs";
    char qmp_dir[100]= "/mnt/socket";

    // File name
    char state_name[100] = "vmtmp_state_"; /*vmtmp_state_${vmid}_${template_num}*/
    strcat(state_name, vmid);
    strcat(state_name, "_");
    strcat(state_name, template_num);
    char mem_name[100] = "vmtmp_mem_"; /*vmtmp_mem_${vmid}_${template_num}*/
    strcat(mem_name, vmid);
    strcat(mem_name, "_");
    strcat(mem_name, template_num);
    char last_mem_name[100] = "vmtmp_mem_"; /*vmtmp_mem_${vmid}_$((${template_num}-1))*/
    strcat(last_mem_name, vmid);
    strcat(last_mem_name, "_");
    strcat(last_mem_name, template_last);
    char disk_name[100] = "vmtmp_disk_"; /*vmtmp_disk_${vmid}_${template_num}.qcow2*/
    strcat(disk_name, vmid);
    strcat(disk_name, "_");
    strcat(disk_name, template_num);
    strcat(disk_name, ".qcow2");
    char qmp_name[100] = "vm_qmp_"; /*vm_qmp_${vmid}.sock*/
    strcat(qmp_name, vmid);
    strcat(qmp_name, ".sock");

    // Check the correctness of the command
    printf("vmid: %s, template_num: %s\n", vmid, template_num);
    int i, len;
    for (i = 0, len = strlen(vmid); i < len; i++) {
        if (!isdigit(vmid[i])) {
            printf("argument 1 is not integer type.\n");
            printf("Input format : templating.sh [vmid] [template_num]\n");
            return;
        }
    }
    for (i = 0, len = strlen(template_num); i < len; i++) {
        if (!isdigit(template_num[i])) {
            printf("argument 2 is not integer type.\n");
            printf("Input format : templating.sh [vmid] [template_num]\n");
            return;
        }
    }
    // Check whether template_num is correct
    FILE *file;
    char mem_path_name[100]; //${mem_dir}/${mem_name}
    strcpy(mem_path_name, mem_dir);
    strcat(mem_path_name, "/");
    strcat(mem_path_name, mem_name);
    char last_mem_path_name[100]; //${mem_dir}/${last_mem_name}
    strcpy(last_mem_path_name, mem_dir);
    strcat(last_mem_path_name, "/");
    strcat(last_mem_path_name, last_mem_name);
    if(file = fopen(mem_path_name, "r")){
        printf("${mem_dir}/${mem_name} exist.\n");
        return;
    }
    else if(template_int != 1 && !(file = fopen(last_mem_path_name, "r"))){
        printf("template_num is not correct.\n");
        return;
    }

    // Get disk name
    char ori_disk_path[512];
    char qmp_path_name[100]; //${qmp_dir}/${qmp_name}
    strcpy(qmp_path_name, qmp_dir);
    strcat(qmp_path_name, "/");
    strcat(qmp_path_name, qmp_name);
    char popen_tmp[1024] = "sudo nc -w 1 -U /mnt/socket/vm_qmp_55.sock | awk 'NR==3' | jq .return[].inserted.image.filename | grep \\\" |sed 's/\\\"//g'";

    BlockInfoList *block_list, *info;
    BlockDeviceInfoList *blockdev_list;
    bool verbose = qdict_get_try_bool(qdict, "verbose", false);
    bool nodes = qdict_get_try_bool(qdict, "nodes", false);
    bool printed = false;
    BlockDeviceInfo *inserted;

    /* Print BlockBackend information */
    if (!nodes) {
        block_list = qmp_query_block(NULL);
    } else {
        block_list = NULL;
    }
    if(template_int == 1){
        for (info = block_list; info; info = info->next) {
            if (info != block_list) {
                monitor_printf(mon, "\n");
            }
            inserted = info->value->has_inserted
                                        ? info->value->inserted : NULL;      
            print_block_info(mon, info->value, inserted, verbose); //去改這裡的source code讓他比對字串

            printed = true;
        }
    }
    printf("ori_disk_path: %s\n", strcpy(ori_disk_path, ori_disk("only return")));

    // Templating
    char state_path_name[100]; //${state_dir}/${state_name}
    strcpy(state_path_name, state_dir);
    strcat(state_path_name, "/");
    strcat(state_path_name, state_name);
    char disk_path_name[100]; //${disk_dir}/${disk_name}
    strcpy(disk_path_name, disk_dir);
    strcat(disk_path_name, "/");
    strcat(disk_path_name, disk_name);

    //template1
    //migrate-set-capabilities 
    const char *cap = "bypass-shared-memory"; //default
    bool state = true; //default
    char msc_tmp[100] = "sudo nc -w 1 -U ";
    strcat(msc_tmp , qmp_path_name);
    Error *err = NULL;

    MigrationCapabilityStatusList *caps = g_malloc0(sizeof(*caps));
    int val;

    val = qapi_enum_parse(&MigrationCapability_lookup, cap, -1, &err);
    if (val < 0) {
        goto end;
    }

    caps->value = g_malloc0(sizeof(*caps->value));
    caps->value->capability = val;
    caps->value->state = state;
    caps->next = NULL;
    qmp_migrate_set_capabilities(caps, &err); //run echo '{"execute":"qmp_capabilities"}{"execute":"migrate-set-capabilities", "arguments":{"capabilities": [{"capability":"bypass-shared-memory", "state":true}]}}'
    int systemRet1 = system(msc_tmp); //run sudo nc -w 1 -U ${qmp_dir}/${qmp_name}
    if(systemRet1 == -1){
    }
    printf("migrate-set-capabilities step is finished !\n");

    //template2
    //query-migrate-capabilities
    MigrationCapabilityStatusList *qcaps, *qcap;
    char qmc_tmp1[100] = " | tail -n 1 | sed 's/{/\\n{/g'";
    char qmc_tmp[100];
    strcpy(qmc_tmp , msc_tmp);
    strcat(qmc_tmp , qmc_tmp1);
    int systemRet2 = system(qmc_tmp); //run sudo nc -w 1 -U ${qmp_dir}/${qmp_name} | tail -n 1 | sed 's/{/\n{/g'
    if(systemRet2 == -1){
    }
    qcaps = qmp_query_migrate_capabilities(NULL);
    if (qcaps) {
        for (qcap = qcaps; qcap; qcap = qcap->next) {
            monitor_printf(mon, "%s: %s\n",
                           MigrationCapability_str(qcap->value->capability),
                           qcap->value->state ? "on" : "off");
        }
    }
    qapi_free_MigrationCapabilityStatusList(qcaps);
    printf("query-migrate-capabilities step is finished !\n");

    //template3
    //migrate
    bool detach = qdict_get_try_bool(qdict, "detach", false);
    bool blk = qdict_get_try_bool(qdict, "blk", false);
    bool inc = qdict_get_try_bool(qdict, "inc", false);
    bool resume = qdict_get_try_bool(qdict, "resume", false);
    bool cuju = qdict_get_try_bool(qdict, "cuju", false);
    bool templating = qdict_get_try_bool(qdict, "templating", true); //
    char uri[100] = "exec:cat > ";
    strcat(uri, state_path_name);
    char mem_path[100];
    strcpy(mem_path, mem_path_name);
    char disk_path[100];
    strcpy(disk_path, disk_path_name);
    qmp_migrate(uri, mem_path, disk_path, !!blk, blk,
                !!inc, inc, false, false, true, resume,
                !!cuju, cuju, !!templating, templating, &err);

    int systemRet3 = system(qmc_tmp); //run sudo nc -w 1 -U ${qmp_dir}/${qmp_name}
    if(systemRet3 == -1){
    }
    printf("migrate step is finished !\n");
    if (!detach && !cuju) {
        HMPMigrationStatus *status;

        /*
        if (monitor_suspend(mon) < 0) {
            monitor_printf(mon, "terminal does not allow synchronous "
                           "migration, continuing detached\n");
            return;
        }
        */

        status = g_malloc0(sizeof(*status));
        status->mon = mon;
        status->is_block_migration = blk || inc;
        status->timer = timer_new_ms(QEMU_CLOCK_REALTIME, hmp_migrate_status_cb,
                                          status);
        timer_mod(status->timer, qemu_clock_get_ms(QEMU_CLOCK_REALTIME));
    }
    
    printf("create tmeplate is finished !\n");

    // Create a symbolic link to original disk image
    char new_disk_name[100] = "vmtmp_disk_";
    char ndn_tmp[100] = "_0.qcow2";
    strcat(new_disk_name, vmid);
    strcat(new_disk_name, ndn_tmp);
    char new_disk_path_name[100]; //${disk_dir}/${new_disk_name}
    strcpy(new_disk_path_name, disk_dir);
    strcat(new_disk_path_name, "/");
    strcat(new_disk_path_name, new_disk_name);
    char lns[1024] = "ln -s "; //ln -s ${ori_disk_path} ${disk_dir}/${new_disk_name}
    strcat(lns, ori_disk_path);
    strcat(lns, " ");
    strcat(lns, new_disk_path_name);
    if(template_int == 1){
        system(lns);
    }
    printf("create a symbolic link is finished !\n");

    if (err) {
        monitor_printf(mon, "%s\n", error_get_pretty(err));
        hmp_handle_error(mon, &err);
        error_free(err);
        return;
    }
end:
    qapi_free_MigrationCapabilityStatusList(caps);
    qapi_free_BlockInfoList(block_list);
    hmp_handle_error(mon, &err);
}

~~* 因為執行hmp_save_all_VMT的qmp_migrate函數會call到對應的qemu_thread_create函數再call到migration/migration.c的函數，中因為變成了multithread所以copy memory page的部份在hmp_save_all_VMT()中可能會亂序，所以把merge_dirty_to_memory.c的程式改增加至save_template_memory_thread()中：~~

#if SEAMLESS_TEMPLATING
static void *save_template_memory_thread(void *opaque)
{
    
    int r;
    r = kvm_save_dirty_memory_to_file();
    if (r < 0) {
        printf("make dirty file error (%d)\n", r);
        monitor_printf(fmin, "make dirty file error (%d)\n", r);
        return NULL;
    }
    printf("finish copied, total pages: %d.\n", r);
    monitor_printf(fmin, "finish copied, total pages: %d.\n", r);
    kvm_close_dirty_file();

    /*merge_dirty_to_memory.c（input: memoryfn , dirtyfn , rwmode）*/
    Error *err = NULL;
    
    const char *memoryfn = "";
    const char *dirtyfn = "";
    const char *rwmode = "";
    char cmemoryfn[50];
    char cdirtyfn[50];
    char crwmode[50];

    const char* filename = "/mnt/nfs/file.txt";
    FILE *in_file = fopen(filename, "r");
    struct stat sb;
    stat(filename, &sb);
    char *file_contents = malloc(sb.st_size);
    int count = 0;
    while (fscanf(in_file, "%[^\n] ", file_contents) != EOF) {
        //printf("> %s\n", file_contents);
        if(count == 0) {
            strcpy(cmemoryfn, file_contents);
            memoryfn = cmemoryfn;
        }
        else if (count == 1){
            strcpy(cdirtyfn, file_contents);
            dirtyfn = cdirtyfn;
        }
        else{
            strcpy(crwmode, file_contents);
            rwmode = crwmode;
        }
        count++;
    }
    fclose(in_file);
    //printf("%s\n%s\n%d\n",memoryfn,dirtyfn,atoi(rwmode)); //test

    qmp_merge_dirty_to_memory(!!memoryfn, memoryfn, !!dirtyfn, dirtyfn, !!rwmode, rwmode, &err);

    printf("merge dirty to memory is finished !\n"); //test
    monitor_printf(fmin, "merge dirty to memory is finished !\n"); //test
    if (err) {
    }

    return NULL;
}

* 在include/monitor/hmp.h新增：
“mon”和“qdict”參數對於所有 HMP 函數都是必需的
前者是監控對象，這樣才能printf到terminal
後者是如何將用戶輸入的參數傳遞給命令實現

void hmp_templating(Monitor *mon, const QDict *qdict);

* 在hmp-commands.hx新增：

{
        {
        .name       = "templating",
        .args_type  = "vmid:s?,template_num:s?",
        .params     = "templating [vmid] [template_num]",
        .help       = "create a vm template file",
        .cmd        = hmp_templating,
    },

STEXI
@item hmp_templating @var{vmid}
@findex templating
create a vm template file
ETEXI

* hmp測試：
1 输入help，就可以看到hmp支持的所有命令
2. 在輸入./create_native_VM_with_templating 55的同個terminal底下輸入hmp 指令：
> templating 55 1
3. ouput(檔案生成，terminal輸出）：

        templating 55 1
    drive0 (#block183): /mnt/nfs/vm_disk_55.qcow2 (qcow2)
        Attached to:      /machine/peripheral-anon/device[0]/virtio-backend
        Cache mode:       writethrough, direct

    ide1-cd0: [not inserted]
        Attached to:      /machine/unattached/device[23]
        Removable device: not locked, tray closed

    floppy0: [not inserted]
        Attached to:      /machine/unattached/device[17]
        Removable device: not locked, tray closed

    sd0: [not inserted]
        Removable device: not locked, tray closed
    {"QMP": {"version": {"qemu": {"micro": 1, "minor": 2, "major": 4}, "package": ""}, "capabilities": ["oob"]}}

    {"QMP": 
    {"version": 
    {"qemu": 
    {"micro": 1, "minor": 2, "major": 4}, "package": ""}, "capabilities": ["oob"]}}
    xbzrle: off
    rdma-pin-all: off
    auto-converge: off
    zero-blocks: off
    compress: off
    events: off
    postcopy-ram: off
    x-colo: off
    cuju-ft: off
    cuju-wdt: off
    release-ram: off
    block: off
    return-path: off
    pause-before-switchover: off
    multifd: off
    dirty-bitmaps: off
    postcopy-blocktime: off
    late-block-activate: off
    x-ignore-shared: off
    bypass-shared-memory: on
    validate-uuid: off
    state: /mnt/state/vmtmp_state_55_1, mem: /mnt/memory/vmtmp_mem_55_1, disk: /mnt/nfs/vmtmp_disk_55_1.qcow2.

    {"QMP": 
    {"version": 
    {"qemu": 
    {"micro": 1, "minor": 2, "major": 4}, "package": ""}, "capabilities": ["oob"]}}
    create tmeplate is finished !
    create a symbolic link is finished !
    Formatting '/mnt/nfs/vmtmp_disk_55_1.qcow2', fmt=qcow2 size=21474836480 backing_file=/mnt/nfs/vm_disk_55.qcow2 backing_fmt=qcow2 cluster_size=65536 lazy_refcounts=off refcount_bits=16
    finish copied, total pages: 524288.

## 新增libvirtd -> qemu driver -> qemu
* 要改的file:
* 流程圖：
### script轉成xml
因為libvirt create VM是透過xml的形式，所以將qemu command的參數轉成xml看懂的形式
* script:

#!/bin/bash

# Input
vmid=$1

# VM hardware settings
vcpu=1
ram=2G
max_vcpu=$(( vcpu - 1 ))
i=1

# File path
mem_dir=/mnt/memory
disk_dir=/mnt/nfs
qmp_dir=/mnt/socket

# File name
mem_name=vm_mem_${vmid}
disk_name=vm_disk_${vmid}.qcow2
qmp_name=vm_qmp_${vmid}.sock

# Check whether vmid is unique
if [ -f ${mem_dir}/${mem_name} ]; then
    echo "${mem_dir}/${mem_name} exist."
    exit
elif [ -f ${qmp_dir}/${qmp_name} ]; then
    echo "${qmp_dir}/${qmp_name} exist."
    exit
fi

# Boot vm
sudo ../x86_64-softmmu/qemu-system-x86_64 \
-cpu host \
-smp ${vcpu},cores=${vcpu},threads=1,sockets=1 \
-m   ${ram},slots=4,maxmem=10240M -enable-kvm \
-object memory-backend-file,id=mem0,size=${ram},mem-path=${mem_dir}/${mem_name},share=on \
-numa node,nodeid=0,cpus=0-${max_vcpu},memdev=mem0 \
-drive if=none,id=drive0,cache=none,format=qcow2,file=${disk_dir}/${disk_name} \
-device virtio-blk,drive=drive0 \
-netdev tap,ifname=tap${vmid},id=mytap,script=no,downscript=no \
-device virtio-net,netdev=mytap \
-vga std -chardev socket,id=mon,path=/home/$(whoami)/vm${vmid}.monitor,server,nowait \
-mon chardev=mon,id=monitor,mode=readline \
-chardev socket,id=qmp,port=4444,host=localhost,server \
-mon chardev=qmp,mode=control,pretty=on

* xml(主要是將無法直接從script轉來xml的參數，透過<qemu:commandline>的方式加進來)

<domain type='kvm' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'>
  <name>create_vmtemplate</name>
  <memory unit='GiB'>1</memory>
  <currentMemory unit='GiB'>1</currentMemory>
  <qemu:commandline>
    <qemu:arg value='-smp'/>
    <qemu:arg value='2,cores=2,threads=1,sockets=1'/>
    <qemu:arg value='-m'/>
    <qemu:arg value='1G,slots=4,maxmem=10240M'/>
    <qemu:arg value='-enable-kvm'/>
    <qemu:arg value='-object'/>
    <qemu:arg value='memory-backend-file,id=mem0,size=1G,mem-path=/mnt/memory/vm_mem_55,share=on'/>
    <qemu:arg value='-numa'/>
    <qemu:arg value='node,nodeid=0,cpus=0-1,memdev=mem0'/>
    <qemu:arg value='-netdev'/>    
    <qemu:arg value='tap,ifname=tap55,id=mytap,script=no,downscript=no'/>
    <qemu:arg value='-device'/>
    <qemu:arg value='virtio-net,netdev=mytap'/>
    <qemu:arg value='-vga'/>
    <qemu:arg value='std'/>
    <qemu:arg value='-chardev'/>
    <qemu:arg value='socket,id=mon,path=/home/cuju42libvirt/vm55.monitor,server,nowait'/>
    <qemu:arg value='-mon'/>
    <qemu:arg value='chardev=mon,id=monitor,mode=readline'/>
    <qemu:arg value='-qmp'/>
    <qemu:arg value='unix:/mnt/socket/vm_qmp_55.sock,server,nowait'/>
    <qemu:arg value='-qmp'/>
    <qemu:arg value='tcp:127.0.0.1:4444,server,nowait'/>
    <qemu:arg value='-monitor'/>
    <qemu:arg value='stdio'/>
  </qemu:commandline>
  <os>
    <type arch='x86_64' machine='pc-i440fx-2.8'>hvm</type>
    <boot dev='hd'/>
  </os>
  <features>
    <acpi/>
    <apic/>
    <vmport state='off'/>
  </features>
  <clock offset='localtime'>
    <timer name='rtc' tickpolicy='catchup'/>
    <timer name='pit' tickpolicy='delay'/>
    <timer name='hpet' present='yes'/>
    <timer name='hypervclock' present='yes'/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <pm>
    <suspend-to-mem enabled='no'/>
    <suspend-to-disk enabled='no'/>
  </pm>
  <devices>
    <emulator>/usr/bin/qemu-system-x86_64</emulator>
    <disk type='file' device='disk'>
      <driver name='qemu' type='qcow2' cache='none'/>
      <source file='/mnt/nfs/vm_disk_55.qcow2'/>
      <target dev='vda' bus='virtio'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
    </disk>
    <controller type='usb' index='0' model='ich9-ehci1'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x7'/>
    </controller>
    <controller type='usb' index='0' model='ich9-uhci1'>
      <master startport='0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0' multifunction='on'/>
    </controller>
    <controller type='usb' index='0' model='ich9-uhci2'>
      <master startport='2'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x1'/>
    </controller>
    <controller type='usb' index='0' model='ich9-uhci3'>
      <master startport='4'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x2'/>
    </controller>
    <controller type='pci' index='0' model='pci-root'/>
    <interface type='bridge'>
      <mac address='52:54:00:d8:f1:59'/>
      <source bridge='br0'/>
      <model type='virtio'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </interface>
    <serial type='pty'>
      <target type='isa-serial' port='0'>
        <model name='isa-serial'/>
      </target>
    </serial>
    <console type='pty'>
      <target type='serial' port='0'/>
    </console>
    <input type='tablet' bus='usb'>
      <address type='usb' bus='0' port='1'/>
    </input>
    <input type='mouse' bus='ps2'/>
    <input type='keyboard' bus='ps2'/>
    <graphics type='vnc' port='-1' autoport='yes'>
      <listen type='address'/>
    </graphics>
    <sound model='ich6'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
    </sound>
    <video>
      <model type='vga' vram='16384' heads='1' primary='yes'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
    </video>
    <redirdev bus='usb' type='spicevmc'>
      <address type='usb' bus='0' port='2'/>
    </redirdev>
    <redirdev bus='usb' type='spicevmc'>
      <address type='usb' bus='0' port='3'/>
    </redirdev>
      <memballoon model='none'/>
  </devices>
</domain>

### 新增libvirtd 函數
可把Libvirt-daemon(API-C)當作一個server，而可以由virsh client來訪問或是再包一層libvirt python api給openstack訪問
* 在libvirt-4.0.0/src/libvirt-qemu.c或是libvirt-domain.c新增：
* 註解一定要寫，參數也要用註解宣告
* 透過flags |= VIR_DOMAIN_QEMU_MONITOR_COMMAND_HMP;
conn->driver->domainQemuMonitorCommand(domain, cmd, &result, flags);利用hmp傳給qemu monitor跟qemue溝通

/**
 * virDomainCreateVmTemplate:
 * @domain: pointer to the domain, or NULL
 * @vmid: id of the virtual machine
 * @template_num: number of the template virtual machine
 * 
 * create VM template, include of device & memory state and disk snapshot
 *
 * Returns 0 on success, -1 on failure
 */
int
virDomainCreateVmTemplate(virDomainPtr domain, const char *vmid, const char *template_num)
{
    virConnectPtr conn;
    const char *cmd;
    char *result = NULL;
    unsigned int flags = 0;
    flags |= VIR_DOMAIN_QEMU_MONITOR_COMMAND_HMP;
    VIR_DOMAIN_DEBUG(domain, "cmd=%s, result=%p, flags=0x%x",
                     cmd, result, flags);

    virResetLastError();
    virCheckDomainReturn(domain, -1);
    conn = domain->conn;

    virCheckReadOnlyGoto(conn->flags, error);

    /*if (!vmid){
        printf("argument 1 is empty\n");
        printf("Input format : templating.sh [vmid] [template_num]\n");
        goto error;
    }
    else if(!template_num){
        printf("argument 2 is empty\n");
        printf("Input format : templating.sh [vmid] [template_num]\n");
        goto error;
    }*/
        
    asprintf(&cmd, "templating %s %s", vmid, template_num);
    printf("cmd:%s\n",cmd);
    if (conn->driver->domainQemuMonitorCommand) {
        int ret;
        ret = conn->driver->domainQemuMonitorCommand(domain, cmd, &result,
                                                     flags);
        if (ret < 0)
            goto error;
        return ret;
    }

    virReportUnsupportedError();

 error:
    virDispatchError(conn);
    return -1;
}

* 在libvirt-4.0.0/include/libvirt/libvirt-qemu.h or libvirt-domain.h新增:

int virDomainCreateVmTemplate(virDomainPtr domain, const char *vmid, const char *template_num);

* 如果不是利用hmp的方式傳遞則須透過qemu-driver，在libvirt-4.0.0/src/qemu/qemu_driver.c新增qemuDomainCreateVmTemplate()函數

## 新增virsh command -> libvirtd
virsh 指令是libvirt內建的command-line inteface(CLI)
### 新增virsh command
* 在libvirt-4.0.0/tools/virsh-domain.c新增：
* 定義virsh templating --help 內容 （.name = "desc"必須加否則會錯誤）：

    //virsh templating --help 內容 .name = "desc"必須加否則會錯誤
    static const vshCmdInfo info_create_vm_template[] = {
        {.name = "help",
         .data = N_("create VM template include of device & memory state and disk snapshot.")
        },
        {.name = "desc",
         .data = N_("create VM template include of device & memory state and disk snapshot.")
        },
        {.name = NULL}
    };

* 定義virsh templating 命令參數：
VIRSH_COMMON_OPT_DOMAIN_FULL(0)代表virsh templating command後會先加domain也就是VM name，EX: virsh templating **create_vmtemplate** 55 1

    //定義virsh templating
    static const vshCmdOptDef opts_create_vm_template[] = {
        VIRSH_COMMON_OPT_DOMAIN_FULL(0),
        {.name = "vmid",
         .type = VSH_OT_STRING,
         .help = N_("id of the virtual machine")
        },
        {.name = "template_num",
         .type = VSH_OT_STRING,
         .help = N_("number of the template virtual machine")
        },
    };

* virsh templating實際函數處理

    //virsh templating實際內容
    static bool 
    cmdCreateVmTemplate(vshControl *ctl, const vshCmd *cmd)
    {
        virDomainPtr dom = NULL;
        bool ret;
        const char *vmid;
        const char *template_num;


        if (!(dom = virshCommandOptDomain(ctl, cmd, NULL)))
            return false;

        if (vshCommandOptStringReq(ctl, cmd, "vmid", &vmid) < 0)
            return false;
        if (vshCommandOptStringReq(ctl, cmd, "template_num", &template_num) < 0)
            return false;

        if (virDomainCreateVmTemplate(dom, vmid, template_num) < 0)
            goto cleanup;
        ret = true;

     cleanup:
        virshDomainFree(dom);
        return ret;
    }

* 註冊virsh templating command

    const vshCmdDef domManagementCmds[] = {
        ....
        {.name = "templating",
         .handler = cmdCreateVmTemplate,
         .opts = opts_create_vm_template,
         .info = info_create_vm_template,
         .flags = 0
        },
        {.name = NULL}
    };

* virsh command 測試

1. virsh define create_vmtemplate.xml
2. virsh start create_vmtemplate
3. virsh templating create_vmtemplate 55 1